Store whole IR.Module in .bindings cache (#8924)

enso-org · Feb 9, 2024 · 9a91b7b · 9a91b7b
1 parent 04e53cc
commit 9a91b7b
Show file tree

Hide file tree

Showing 23 changed files with 1,113 additions and 1,090 deletions.
diff --git a/engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java b/engine/runtime-compiler/src/main/java/org/enso/compiler/context/CompilerContext.java
@@ -104,10 +104,13 @@ Future<Boolean> serializeLibrary(
       Compiler compiler, LibraryName libraryName, boolean useGlobalCacheLocations);
 
   Future<Boolean> serializeModule(
-      Compiler compiler, Module module, boolean useGlobalCacheLocations);
+      Compiler compiler, Module module, boolean useGlobalCacheLocations, boolean usePool);
 
   boolean deserializeModule(Compiler compiler, Module module);
 
+  scala.Option<List<org.enso.polyglot.Suggestion>> deserializeSuggestions(LibraryName libraryName)
+      throws InterruptedException;
+
   void shutdown(boolean waitForPendingJobCompletion);
 
   public static interface Updater {

diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/Compiler.scala
@@ -24,7 +24,7 @@ import org.enso.compiler.core.ir.module.scope.Export
 import org.enso.compiler.core.ir.module.scope.Import
 import org.enso.compiler.core.ir.module.scope.imports
 import org.enso.compiler.core.EnsoParser
-import org.enso.compiler.data.{BindingsMap, CompilerConfig}
+import org.enso.compiler.data.CompilerConfig
 import org.enso.compiler.exception.CompilationAbortedException
 import org.enso.compiler.pass.PassManager
 import org.enso.compiler.pass.analyse._
@@ -200,7 +200,11 @@ class Compiler(
     */
   def generateDocs(module: Module): Module = {
     initialize()
-    parseModule(module, isGenDocs = true)
+    parseModule(
+      module,
+      irCachingEnabled && !context.isInteractive(module),
+      isGenDocs = true
+    )
     module
   }
 
@@ -244,7 +248,7 @@ class Compiler(
     initialize()
     modules.foreach(m =>
       try {
-        parseModule(m)
+        parseModule(m, irCachingEnabled && !context.isInteractive(m))
       } catch {
         case e: Throwable =>
           context.log(
@@ -286,12 +290,12 @@ class Compiler(
           )
         )
         context.updateModule(module, _.invalidateCache())
-        parseModule(module)
+        parseModule(module, irCachingEnabled && !context.isInteractive(module))
         importedModules
           .filter(isLoadedFromSource)
           .map(m => {
             if (m.getBindingsMap() == null) {
-              parseModule(m)
+              parseModule(m, irCachingEnabled && !context.isInteractive(module))
             }
           })
         runImportsAndExportsResolution(module, generateCode)
@@ -302,7 +306,7 @@ class Compiler(
 
     if (irCachingEnabled) {
       requiredModules.foreach { module =>
-        ensureParsed(module)
+        ensureParsed(module, !context.isInteractive(module))
       }
     }
     requiredModules.foreach { module =>
@@ -404,6 +408,7 @@ class Compiler(
 
         if (shouldCompileDependencies || isModuleInRootPackage(module)) {
           val shouldStoreCache =
+            generateCode &&
             irCachingEnabled && !context.wasLoadedFromCache(module)
           if (
             shouldStoreCache && !hasErrors(module) &&
@@ -415,7 +420,8 @@ class Compiler(
               context.serializeModule(
                 this,
                 module,
-                useGlobalCacheLocations
+                useGlobalCacheLocations,
+                true
               )
             }
           }
@@ -481,7 +487,7 @@ class Compiler(
 
   private def ensureParsedAndAnalyzed(module: Module): Unit = {
     if (module.getBindingsMap() == null) {
-      ensureParsed(module)
+      ensureParsed(module, irCachingEnabled && !context.isInteractive(module))
     }
     if (context.isSynthetic(module)) {
       // Synthetic modules need to be import-analyzed
@@ -490,19 +496,10 @@ class Compiler(
       // TODO: consider generating IR for synthetic modules, if possible.
       importExportBindings(module) match {
         case Some(bindings) =>
-          val converted = bindings
-            .toConcrete(packageRepository.getModuleMap)
-            .map { concreteBindings =>
-              concreteBindings
-            }
-          ensureParsed(module)
-          val currentLocal = module.getBindingsMap()
-          currentLocal.resolvedImports =
-            converted.map(_.resolvedImports).getOrElse(Nil)
-          currentLocal.resolvedExports =
-            converted.map(_.resolvedExports).getOrElse(Nil)
-          currentLocal.exportedSymbols =
-            converted.map(_.exportedSymbols).getOrElse(Map.empty)
+          context.updateModule(
+            module,
+            _.ir(bindings)
+          )
         case _ =>
       }
     }
@@ -520,7 +517,7 @@ class Compiler(
     * @param module - the scope from which docs are generated.
     */
   def gatherImportStatements(module: Module): Array[String] = {
-    ensureParsed(module)
+    ensureParsed(module, irCachingEnabled && !context.isInteractive(module))
     val importedModules = context.getIr(module).imports.flatMap {
       case imp: Import.Module =>
         imp.name.parts.take(2).map(_.name) match {
@@ -543,6 +540,7 @@ class Compiler(
 
   private def parseModule(
     module: Module,
+    useCaches: Boolean,
     isGenDocs: Boolean = false
   ): Unit = {
     context.log(
@@ -552,7 +550,7 @@ class Compiler(
     )
     context.updateModule(module, _.resetScope())
 
-    if (irCachingEnabled && !context.isInteractive(module)) {
+    if (useCaches) {
       if (context.deserializeModule(this, module)) {
         return
       }
@@ -566,7 +564,7 @@ class Compiler(
     * @param module module which is conssidered
     * @return module's bindings, if available in libraries' bindings cache
     */
-  def importExportBindings(module: Module): Option[BindingsMap] = {
+  def importExportBindings(module: Module): Option[IRModule] = {
     if (irCachingEnabled && !context.isInteractive(module)) {
       val libraryName = Option(module.getPackage).map(_.libraryName)
       libraryName.flatMap(
@@ -646,14 +644,19 @@ class Compiler(
     * @param module the module to ensure is parsed.
     */
   def ensureParsed(module: Module): Unit = {
+    val useCaches = irCachingEnabled && !context.isInteractive(module)
+    ensureParsed(module, useCaches)
+  }
+
+  def ensureParsed(module: Module, useCaches: Boolean): Unit = {
     if (
       !context
         .getCompilationStage(module)
         .isAtLeast(
           CompilationStage.AFTER_PARSING
         )
     ) {
-      parseModule(module)
+      parseModule(module, useCaches)
     }
   }
 

diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/PackageRepository.scala
@@ -3,7 +3,7 @@ package org.enso.compiler
 import com.oracle.truffle.api.TruffleFile
 import org.enso.editions.LibraryName
 import org.enso.compiler.context.CompilerContext
-import org.enso.compiler.data.BindingsMap
+import org.enso.compiler.core.ir.{Module => IRModule}
 import org.enso.pkg.{ComponentGroups, Package, QualifiedName}
 
 import scala.collection.immutable.ListSet
@@ -113,7 +113,7 @@ trait PackageRepository {
     libraryName: LibraryName,
     moduleName: QualifiedName,
     context: CompilerContext
-  ): Option[BindingsMap]
+  ): Option[IRModule]
 
 }
 

diff --git a/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala b/engine/runtime-compiler/src/main/scala/org/enso/compiler/phase/ImportResolver.scala
@@ -66,7 +66,7 @@ class ImportResolver(compiler: Compiler) {
                 u.invalidateCache()
               }
             )
-            compiler.ensureParsed(current)
+            compiler.ensureParsed(current, false)
             return analyzeModule(current)
         }
       // put the list of resolved imports in the module metadata
@@ -123,18 +123,14 @@ class ImportResolver(compiler: Compiler) {
             // - no - ensure they are parsed (load them from cache) and add them to the import/export resolution
             compiler.importExportBindings(current) match {
               case Some(bindings) =>
-                val converted = bindings
-                  .toConcrete(compiler.packageRepository.getModuleMap)
-                  .map { concreteBindings =>
-                    compiler.context.updateModule(
-                      current,
-                      { u =>
-                        u.bindingsMap(concreteBindings)
-                        u.loadedFromCache(true)
-                      }
-                    )
-                    concreteBindings
+                compiler.context.updateModule(
+                  current,
+                  u => {
+                    u.ir(bindings)
+                    u.loadedFromCache(true)
                   }
+                )
+                val converted = Option(current.getBindingsMap())
                 (
                   converted
                     .map(
@@ -183,7 +179,7 @@ class ImportResolver(compiler: Compiler) {
             u.compilationStage(CompilationStage.INITIAL)
           }
         )
-        compiler.ensureParsed(mod)
+        compiler.ensureParsed(mod, false)
         b = mod.getBindingsMap()
       }
 

diff --git a/...strument-common/src/main/java/org/enso/interpreter/instrument/job/SerializeModuleJob.java b/...strument-common/src/main/java/org/enso/interpreter/instrument/job/SerializeModuleJob.java
@@ -2,7 +2,6 @@
 
 import java.util.logging.Level;
 import org.enso.interpreter.instrument.execution.RuntimeContext;
-import org.enso.interpreter.runtime.SerializationManager;
 import org.enso.pkg.QualifiedName;
 import org.enso.polyglot.CompilationStage;
 
@@ -22,7 +21,6 @@ public SerializeModuleJob(QualifiedName moduleName) {
   public Void run(RuntimeContext ctx) {
     var ensoContext = ctx.executionService().getContext();
     var compiler = ensoContext.getCompiler();
-    SerializationManager serializationManager = SerializationManager.apply(compiler.context());
     boolean useGlobalCacheLocations = ensoContext.isUseGlobalCache();
     var writeLockTimestamp = ctx.locking().acquireWriteCompilationLock();
     try {
@@ -40,9 +38,10 @@ public Void run(RuntimeContext ctx) {
                           new Object[] {module.getName(), module.getCompilationStage()});
                   return;
                 }
-
-                serializationManager.serializeModule(
-                    compiler, module.asCompilerModule(), useGlobalCacheLocations, false);
+                compiler
+                    .context()
+                    .serializeModule(
+                        compiler, module.asCompilerModule(), useGlobalCacheLocations, false);
               });
     } finally {
       ctx.locking().releaseWriteCompilationLock();

diff --git a/...src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala b/...src/main/scala/org/enso/interpreter/instrument/job/DeserializeLibrarySuggestionsJob.scala
@@ -1,7 +1,6 @@
 package org.enso.interpreter.instrument.job
 
 import org.enso.editions.LibraryName
-import org.enso.interpreter.runtime.SerializationManager
 import org.enso.interpreter.instrument.execution.RuntimeContext
 import org.enso.polyglot.runtime.Runtime.Api
 
@@ -33,17 +32,14 @@ final class DeserializeLibrarySuggestionsJob(
       "Deserializing suggestions for library [{}].",
       libraryName
     )
-    val serializationManager = SerializationManager(
-      ctx.executionService.getContext.getCompiler.context
-    )
-    serializationManager
+    ctx.executionService.getContext.getCompiler.context
       .deserializeSuggestions(libraryName)
       .foreach { cachedSuggestions =>
         ctx.endpoint.sendToClient(
           Api.Response(
             Api.SuggestionsDatabaseSuggestionsLoadedNotification(
               libraryName,
-              cachedSuggestions.getSuggestions.asScala.toVector
+              cachedSuggestions.asScala.toVector
             )
           )
         )