Fix deadlock in initialization of CoreBTypes using Lazy container (#1…

…9298)

Replaces #19297 and fixes #19293 

The deadlocks are now fixed by introduction of
`PostProcessorFrontendAccess.Lazy[T]` container for which initialization
is synchronized with the frontend Context, while providing a thread-safe
access lacking in original solution.

It now also audits where the unrestricted access to context was used and
replaces these usages with safer access.
Reverts #19292
[Cherry-picked 33bdaac][modified]

compiler/src/dotty/tools/backend/jvm/BTypes.scala

@@ -16,8 +16,6 @@ import scala.tools.asm
  */
 abstract class BTypes { self =>
   val frontendAccess: PostProcessorFrontendAccess
-  import frontendAccess.{frontendSynch}
-
   val int: DottyBackendInterface
   import int.given
   /**

compiler/src/dotty/tools/backend/jvm/CodeGen.scala

@@ -37,7 +37,6 @@ import dotty.tools.dotc.util.NoSourcePosition
 class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)( val bTypes: BTypesFromSymbols[int.type]) { self =>
   import DottyBackendInterface.symExtensions
   import bTypes.*
-  import int.given
 
   private lazy val mirrorCodeGen = Impl.JMirrorBuilder()
 
@@ -124,7 +123,7 @@ class CodeGen(val int: DottyBackendInterface, val primitives: DottyPrimitives)(
   }
 
   // Creates a callback that will be evaluated in PostProcessor after creating a file
-  private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: util.SourceFile): AbstractFile => Unit = {
+  private def onFileCreated(cls: ClassNode, claszSymbol: Symbol, sourceFile: util.SourceFile)(using Context): AbstractFile => Unit = {
     val (fullClassName, isLocal) = atPhase(sbtExtractDependenciesPhase) {
       (ExtractDependencies.classNameAsString(claszSymbol), claszSymbol.isLocal)
     }

compiler/src/dotty/tools/backend/jvm/CoreBTypes.scala

@@ -8,6 +8,7 @@ import dotty.tools.dotc.transform.Erasure
 import scala.tools.asm.{Handle, Opcodes}
 import dotty.tools.dotc.core.StdNames
 import BTypes.InternalName
+import PostProcessorFrontendAccess.Lazy
 
 abstract class CoreBTypes {
   val bTypes: BTypes
@@ -56,16 +57,16 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
   val bTypes: BTypesFromSymbols[I]
 
   import bTypes.*
-  import int.given
   import DottyBackendInterface.*
-  import frontendAccess.frontendSynch
   import dotty.tools.dotc.core.Contexts.Context
-
+  import frontendAccess.perRunLazy
   /**
    * Maps primitive types to their corresponding PrimitiveBType. The map is defined lexically above
    * the first use of `classBTypeFromSymbol` because that method looks at the map.
    */
-  lazy val primitiveTypeMap: Map[Symbol, PrimitiveBType] = Map(
+  override def primitiveTypeMap: Map[Symbol, bTypes.PrimitiveBType] = _primitiveTypeMap.get
+  private lazy val _primitiveTypeMap: Lazy[Map[Symbol, PrimitiveBType]] = perRunLazy:
+    Map(
     defn.UnitClass    -> UNIT,
     defn.BooleanClass -> BOOL,
     defn.CharClass    -> CHAR,
@@ -81,7 +82,8 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
    * Map from primitive types to their boxed class type. Useful when pushing class literals onto the
    * operand stack (ldc instruction taking a class literal), see genConstant.
    */
-  lazy val boxedClassOfPrimitive: Map[PrimitiveBType, ClassBType] = frontendSynch(Map(
+  override def boxedClassOfPrimitive: Map[PrimitiveBType, ClassBType] = _boxedClassOfPrimitive.get
+  private lazy val _boxedClassOfPrimitive: Lazy[Map[PrimitiveBType, ClassBType]] = perRunLazy(Map(
       UNIT   -> classBTypeFromSymbol(requiredClass[java.lang.Void]),
       BOOL   -> classBTypeFromSymbol(requiredClass[java.lang.Boolean]),
       BYTE   -> classBTypeFromSymbol(requiredClass[java.lang.Byte]),
@@ -99,7 +101,8 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
    * Maps the method symbol for a box method to the boxed type of the result. For example, the
    * method symbol for `Byte.box()` is mapped to the ClassBType `java/lang/Byte`.
    */
-  lazy val boxResultType: Map[Symbol, ClassBType] = {
+  override def boxResultType: Map[Symbol, ClassBType] = _boxResultType.get
+  private lazy val _boxResultType: Lazy[Map[Symbol, ClassBType]] = perRunLazy{
     val boxMethods = defn.ScalaValueClasses().map{x => // @darkdimius Are you sure this should be a def?
       (x, Erasure.Boxing.boxMethod(x.asClass))
     }.toMap
@@ -110,17 +113,14 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
   /**
    * Maps the method symbol for an unbox method to the primitive type of the result.
    * For example, the method symbol for `Byte.unbox()`) is mapped to the PrimitiveBType BYTE. */
-  lazy val unboxResultType: Map[Symbol, PrimitiveBType] = {
+  override def unboxResultType: Map[Symbol, PrimitiveBType] = _unboxResultType.get
+  private lazy val _unboxResultType = perRunLazy[Map[Symbol, PrimitiveBType]]{
     val unboxMethods: Map[Symbol, Symbol] =
       defn.ScalaValueClasses().map(x => (x, Erasure.Boxing.unboxMethod(x.asClass))).toMap
     for ((valueClassSym, unboxMethodSym) <- unboxMethods)
     yield unboxMethodSym -> primitiveTypeMap(valueClassSym)
   }
 
-  // Used to synchronize initialization of Context dependent ClassBTypes which can be accessed from multiple-threads
-  // Unsychronized initialization might lead errors in either CodeGen or PostProcessor
-  inline private def synchClassBTypeFromSymbol(inline sym: Symbol) = frontendSynch(classBTypeFromSymbol(sym))
-
   /*
    * srNothingRef and srNullRef exist at run-time only. They are the bytecode-level manifestation (in
    * method signatures only) of what shows up as NothingClass (scala.Nothing) resp. NullClass (scala.Null) in Scala ASTs.
@@ -129,35 +129,76 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
    * names of NothingClass and NullClass can't be emitted as-is.
    * TODO @lry Once there's a 2.11.3 starr, use the commented argument list. The current starr crashes on the type literal `scala.runtime.Nothing$`
    */
-  lazy val srNothingRef : ClassBType = synchClassBTypeFromSymbol(requiredClass("scala.runtime.Nothing$"))
-  lazy val srNullRef    : ClassBType = synchClassBTypeFromSymbol(requiredClass("scala.runtime.Null$"))
-
-  lazy val ObjectRef : ClassBType = synchClassBTypeFromSymbol(defn.ObjectClass)
-  lazy val StringRef : ClassBType = synchClassBTypeFromSymbol(defn.StringClass)
-
-  lazy val jlStringBuilderRef       : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.StringBuilder])
-  lazy val jlStringBufferRef        : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.StringBuffer])
-  lazy val jlCharSequenceRef        : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.CharSequence])
-  lazy val jlClassRef               : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.Class[_]])
-  lazy val jlThrowableRef           : ClassBType = synchClassBTypeFromSymbol(defn.ThrowableClass)
-  lazy val jlCloneableRef           : ClassBType = synchClassBTypeFromSymbol(defn.JavaCloneableClass)
-  lazy val jiSerializableRef        : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.io.Serializable])
-  lazy val jlClassCastExceptionRef  : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.ClassCastException])
-  lazy val jlIllegalArgExceptionRef : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.IllegalArgumentException])
-  lazy val jliSerializedLambdaRef   : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.invoke.SerializedLambda])
-
-  lazy val srBoxesRuntimeRef: ClassBType = synchClassBTypeFromSymbol(requiredClass[scala.runtime.BoxesRunTime])
-
-  private lazy val jliCallSiteRef            : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.invoke.CallSite])
-  private lazy val jliLambdaMetafactoryRef   : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.invoke.LambdaMetafactory])
-  private lazy val jliMethodHandleRef        : ClassBType = synchClassBTypeFromSymbol(defn.MethodHandleClass)
-  private lazy val jliMethodHandlesLookupRef : ClassBType = synchClassBTypeFromSymbol(defn.MethodHandlesLookupClass)
-  private lazy val jliMethodTypeRef          : ClassBType = synchClassBTypeFromSymbol(requiredClass[java.lang.invoke.MethodType])
-  private lazy val jliStringConcatFactoryRef : ClassBType = synchClassBTypeFromSymbol(requiredClass("java.lang.invoke.StringConcatFactory")) // since JDK 9
-
-  lazy val srLambdaDeserialize               : ClassBType = synchClassBTypeFromSymbol(requiredClass[scala.runtime.LambdaDeserialize])
-
-  lazy val jliLambdaMetaFactoryMetafactoryHandle = frontendSynch{ new Handle(
+  override def srNothingRef: ClassBType = _srNothingRef.get
+  private lazy val _srNothingRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass("scala.runtime.Nothing$")))
+
+  override def srNullRef: ClassBType = _srNullRef.get
+  private lazy val _srNullRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass("scala.runtime.Null$")))
+
+  override def ObjectRef: ClassBType = _ObjectRef.get
+  private lazy val _ObjectRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(defn.ObjectClass))
+
+  override def StringRef: ClassBType = _StringRef.get
+  private lazy val _StringRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(defn.StringClass))
+
+  override def jlStringBuilderRef: ClassBType = _jlStringBuilderRef.get
+  private lazy val _jlStringBuilderRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.StringBuilder]))
+
+  override def jlStringBufferRef: ClassBType = _jlStringBufferRef.get
+  private lazy val _jlStringBufferRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.StringBuffer]))
+
+  override def jlCharSequenceRef: ClassBType = _jlCharSequenceRef.get
+  private lazy val _jlCharSequenceRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.CharSequence]))
+
+  override def jlClassRef: ClassBType = _jlClassRef.get
+  private lazy val _jlClassRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.Class[?]]))
+
+  override def jlThrowableRef: ClassBType = _jlThrowableRef.get
+  private lazy val _jlThrowableRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(defn.ThrowableClass))
+
+  override def jlCloneableRef: ClassBType = _jlCloneableRef.get
+  private lazy val _jlCloneableRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(defn.JavaCloneableClass))
+
+  override def jiSerializableRef: ClassBType = _jiSerializableRef.get
+  private lazy val _jiSerializableRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.io.Serializable]))
+
+  override def jlClassCastExceptionRef: ClassBType = _jlClassCastExceptionRef.get
+  private lazy val _jlClassCastExceptionRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.ClassCastException]))
+
+  override def jlIllegalArgExceptionRef: ClassBType = _jlIllegalArgExceptionRef.get
+  private lazy val _jlIllegalArgExceptionRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.IllegalArgumentException]))
+
+  override def jliSerializedLambdaRef: ClassBType = _jliSerializedLambdaRef.get
+  private lazy val _jliSerializedLambdaRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.invoke.SerializedLambda]))
+
+  override def srBoxesRuntimeRef: ClassBType = _srBoxesRuntimeRef.get
+  private lazy val _srBoxesRuntimeRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[scala.runtime.BoxesRunTime]))
+
+  private def jliCallSiteRef: ClassBType = _jliCallSiteRef.get
+  private lazy val _jliCallSiteRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.invoke.CallSite]))
+
+  private def jliLambdaMetafactoryRef: ClassBType = _jliLambdaMetafactoryRef.get
+  private lazy val _jliLambdaMetafactoryRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.invoke.LambdaMetafactory]))
+
+  private def jliMethodHandleRef: ClassBType = _jliMethodHandleRef.get
+  private lazy val _jliMethodHandleRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(defn.MethodHandleClass))
+
+  private def jliMethodHandlesLookupRef: ClassBType = _jliMethodHandlesLookupRef.get
+  private lazy val _jliMethodHandlesLookupRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(defn.MethodHandlesLookupClass))
+
+  private def jliMethodTypeRef: ClassBType = _jliMethodTypeRef.get
+  private lazy val _jliMethodTypeRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[java.lang.invoke.MethodType]))
+
+  // since JDK 9
+  private def jliStringConcatFactoryRef: ClassBType = _jliStringConcatFactoryRef.get
+  private lazy val _jliStringConcatFactoryRef: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass("java.lang.invoke.StringConcatFactory")))
+
+  private def srLambdaDeserialize: ClassBType = _srLambdaDeserialize.get
+  private lazy val _srLambdaDeserialize: Lazy[ClassBType] = perRunLazy(classBTypeFromSymbol(requiredClass[scala.runtime.LambdaDeserialize]))
+
+
+  override def jliLambdaMetaFactoryMetafactoryHandle = _jliLambdaMetaFactoryMetafactoryHandle.get
+  private lazy val _jliLambdaMetaFactoryMetafactoryHandle: Lazy[Handle] = perRunLazy{new Handle(
     Opcodes.H_INVOKESTATIC,
     jliLambdaMetafactoryRef.internalName,
     "metafactory",
@@ -167,7 +208,8 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
     ).descriptor,
     /* itf = */ false)}
 
-  lazy val jliLambdaMetaFactoryAltMetafactoryHandle = frontendSynch{ new Handle(
+  override def jliLambdaMetaFactoryAltMetafactoryHandle = _jliLambdaMetaFactoryAltMetafactoryHandle.get
+  private lazy val _jliLambdaMetaFactoryAltMetafactoryHandle: Lazy[Handle] = perRunLazy{ new Handle(
     Opcodes.H_INVOKESTATIC,
     jliLambdaMetafactoryRef.internalName,
     "altMetafactory",
@@ -177,7 +219,8 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
     ).descriptor,
     /* itf = */ false)}
 
-  lazy val jliLambdaDeserializeBootstrapHandle: Handle = frontendSynch{ new Handle(
+  override def jliLambdaDeserializeBootstrapHandle: Handle = _jliLambdaDeserializeBootstrapHandle.get
+  private lazy val _jliLambdaDeserializeBootstrapHandle: Lazy[Handle] = perRunLazy{ new Handle(
     Opcodes.H_INVOKESTATIC,
     srLambdaDeserialize.internalName,
     "bootstrap",
@@ -187,7 +230,8 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
     ).descriptor,
     /* itf = */ false)}
 
-  lazy val jliStringConcatFactoryMakeConcatWithConstantsHandle = frontendSynch{ new Handle(
+  override def jliStringConcatFactoryMakeConcatWithConstantsHandle = _jliStringConcatFactoryMakeConcatWithConstantsHandle.get
+  private lazy val _jliStringConcatFactoryMakeConcatWithConstantsHandle: Lazy[Handle] = perRunLazy{ new Handle(
     Opcodes.H_INVOKESTATIC,
     jliStringConcatFactoryRef.internalName,
     "makeConcatWithConstants",
@@ -199,6 +243,7 @@ abstract class CoreBTypesFromSymbols[I <: DottyBackendInterface] extends CoreBTy
 
   /**
    * Methods in scala.runtime.BoxesRuntime
+   * No need to wrap in Lazy to synchronize access, symbols won't change
    */
   lazy val asmBoxTo  : Map[BType, MethodNameAndType] = Map(
     BOOL   -> MethodNameAndType("boxToBoolean",   MethodBType(List(BOOL),   boxedClassOfPrimitive(BOOL))),

compiler/src/dotty/tools/backend/jvm/GenBCode.scala

@@ -75,7 +75,7 @@ class GenBCode extends Phase { self =>
   }
 
   override def run(using Context): Unit =
-    frontendAccess.frontendSynch {
+    frontendAccess.frontendSynchWithoutContext {
       backendInterface.ctx
       .asInstanceOf[FreshContext]
       .setCompilationUnit(ctx.compilationUnit)

compiler/src/dotty/tools/backend/jvm/PostProcessorFrontendAccess.scala

@@ -8,12 +8,13 @@ import java.util.{Collection => JCollection, Map => JMap}
 import dotty.tools.dotc.core.Contexts.Context
 import dotty.tools.dotc.report
 import dotty.tools.dotc.core.Phases
+import scala.compiletime.uninitialized
 
 /**
  * Functionality needed in the post-processor whose implementation depends on the compiler
  * frontend. All methods are synchronized.
  */
-sealed abstract class PostProcessorFrontendAccess {
+sealed abstract class PostProcessorFrontendAccess(backendInterface: DottyBackendInterface) {
   import PostProcessorFrontendAccess.*
 
   def compilerSettings: CompilerSettings
@@ -25,10 +26,29 @@ sealed abstract class PostProcessorFrontendAccess {
   def getEntryPoints: List[String]
 
   private val frontendLock: AnyRef = new Object()
-  inline final def frontendSynch[T](inline x: T): T = frontendLock.synchronized(x)
+  inline final def frontendSynch[T](inline x: Context ?=> T): T = frontendLock.synchronized(x(using backendInterface.ctx))
+  inline final def frontendSynchWithoutContext[T](inline x: T): T = frontendLock.synchronized(x)
+  inline def perRunLazy[T](inline init: Context ?=> T): Lazy[T] = new Lazy(init)(using this)
 }
 
 object PostProcessorFrontendAccess {
+  /* A container for value with lazy initialization synchronized on compiler frontend
+   * Used for sharing variables requiring a Context for initialization, between different threads
+   * Similar to Scala 2 BTypes.LazyVar, but without re-initialization of BTypes.LazyWithLock. These were not moved to PostProcessorFrontendAccess only due to problematic architectural decisions.
+   */
+  class Lazy[T](init: Context ?=> T)(using frontendAccess: PostProcessorFrontendAccess) {
+    @volatile private var isInit: Boolean = false
+    private var v: T = uninitialized
+
+    def get: T =
+      if isInit then v
+      else frontendAccess.frontendSynch {
+        if !isInit then v = init
+        isInit = true
+        v
+    }
+  }
+
   sealed trait CompilerSettings {
     def debug: Boolean
     def target: String // javaOutputVersion
@@ -79,16 +99,16 @@ object PostProcessorFrontendAccess {
   }
 
 
-  class Impl[I <: DottyBackendInterface](val int: I, entryPoints: HashSet[String])(using ctx: Context) extends PostProcessorFrontendAccess {
-    lazy val compilerSettings: CompilerSettings = buildCompilerSettings()
+  class Impl[I <: DottyBackendInterface](int: I, entryPoints: HashSet[String]) extends PostProcessorFrontendAccess(int) {
+    override def compilerSettings: CompilerSettings = _compilerSettings.get
+    private lazy val _compilerSettings: Lazy[CompilerSettings] = perRunLazy(buildCompilerSettings)
 
-    private def buildCompilerSettings(): CompilerSettings = new CompilerSettings {
+    private def buildCompilerSettings(using ctx: Context): CompilerSettings = new CompilerSettings {
       extension [T](s: dotty.tools.dotc.config.Settings.Setting[T])
-         def valueSetByUser: Option[T] =
-           Option(s.value).filter(_ != s.default)
-      def s = ctx.settings
+         def valueSetByUser: Option[T] = Option(s.value).filter(_ != s.default)
+      inline def s = ctx.settings
 
-      lazy val target =
+      override val target =
         val releaseValue = Option(s.javaOutputVersion.value).filter(_.nonEmpty)
         val targetValue = Option(s.XuncheckedJavaOutputVersion.value).filter(_.nonEmpty)
         (releaseValue, targetValue) match
@@ -99,13 +119,13 @@ object PostProcessorFrontendAccess {
             release
           case (None, None) => "8" // least supported version by default
 
-      lazy val debug: Boolean = ctx.debug
-      lazy val dumpClassesDirectory: Option[String] = s.Ydumpclasses.valueSetByUser
-      lazy val outputDirectory: AbstractFile = s.outputDir.value
-      lazy val mainClass: Option[String] = s.XmainClass.valueSetByUser
-      lazy val jarCompressionLevel: Int = s.YjarCompressionLevel.value
-      lazy val backendParallelism: Int = s.YbackendParallelism.value
-      lazy val backendMaxWorkerQueue: Option[Int] = s.YbackendWorkerQueue.valueSetByUser
+      override val debug: Boolean = ctx.debug
+      override val dumpClassesDirectory: Option[String] = s.Ydumpclasses.valueSetByUser
+      override val outputDirectory: AbstractFile = s.outputDir.value
+      override val mainClass: Option[String] = s.XmainClass.valueSetByUser
+      override val jarCompressionLevel: Int = s.YjarCompressionLevel.value
+      override val backendParallelism: Int = s.YbackendParallelism.value
+      override val backendMaxWorkerQueue: Option[Int] = s.YbackendWorkerQueue.valueSetByUser
      }
 
      private lazy val localReporter = new ThreadLocal[BackendReporting]
@@ -125,7 +145,7 @@ object PostProcessorFrontendAccess {
        else local.nn
      }
 
-    object directBackendReporting extends BackendReporting {
+    override object directBackendReporting extends BackendReporting {
       def error(message: Context ?=> Message, position: SourcePosition): Unit = frontendSynch(report.error(message, position))
       def warning(message: Context ?=> Message, position: SourcePosition): Unit = frontendSynch(report.warning(message, position))
       def log(message: String): Unit = frontendSynch(report.log(message))