Ensure slow shutdown of LS always kicks off hooks

lib/scala/project-manager/src/main/scala/org/enso/projectmanager/infrastructure/languageserver/LanguageServerController.scala

@@ -297,6 +297,9 @@ class LanguageServerController(
           s"Received client ($clientId) disconnect request during shutdown. Ignoring."
         )
 
+      case ShutDownServer =>
+        logger.debug(s"Received shutdown request during shutdown. Ignoring.")
+
       case m: StartServer =>
         // This instance has not yet been shut down. Retry
         context.parent.forward(Retry(m))

lib/scala/project-manager/src/main/scala/org/enso/projectmanager/infrastructure/languageserver/LanguageServerKiller.scala

@@ -2,6 +2,7 @@ package org.enso.projectmanager.infrastructure.languageserver
 
 import akka.actor.{Actor, ActorRef, Cancellable, PoisonPill, Props, Terminated}
 import com.typesafe.scalalogging.LazyLogging
+import org.enso.projectmanager.event.ProjectEvent.ProjectClosed
 import org.enso.projectmanager.infrastructure.languageserver.LanguageServerController.ShutDownServer
 import org.enso.projectmanager.infrastructure.languageserver.LanguageServerKiller.KillTimeout
 import org.enso.projectmanager.infrastructure.languageserver.LanguageServerProtocol.{
@@ -10,6 +11,7 @@ import org.enso.projectmanager.infrastructure.languageserver.LanguageServerProto
 }
 import org.enso.projectmanager.util.UnhandledLogging
 
+import java.util.UUID
 import scala.concurrent.duration.FiniteDuration
 
 /** An actor that shuts all running language servers. It orchestrates all
@@ -20,7 +22,7 @@ import scala.concurrent.duration.FiniteDuration
   * @param shutdownTimeout a shutdown timeout
   */
 class LanguageServerKiller(
-  controllers: List[ActorRef],
+  controllers: Map[UUID, ActorRef],
   shutdownTimeout: FiniteDuration
 ) extends Actor
     with LazyLogging
@@ -34,20 +36,22 @@ class LanguageServerKiller(
       context.stop(self)
     } else {
       logger.info("Killing all servers [{}].", controllers)
-      controllers.foreach(context.watch)
-      controllers.foreach(_ ! ShutDownServer)
+      controllers.foreach { case (_, ref) =>
+        context.watch(ref)
+        ref ! ShutDownServer
+      }
       val cancellable =
         context.system.scheduler.scheduleOnce(
           shutdownTimeout,
           self,
           KillTimeout
         )
-      context.become(killing(controllers.toSet, cancellable, sender()))
+      context.become(killing(controllers.map(_.swap), cancellable, sender()))
     }
   }
 
   private def killing(
-    liveControllers: Set[ActorRef],
+    liveControllers: Map[ActorRef, UUID],
     cancellable: Cancellable,
     replyTo: ActorRef
   ): Receive = {
@@ -63,7 +67,13 @@ class LanguageServerKiller(
       }
 
     case KillTimeout =>
-      liveControllers.foreach(_ ! PoisonPill)
+      logger.warn(
+        s"Not all language servers' controllers finished on time. Forcing termination."
+      )
+      liveControllers.foreach { case (actorRef, projectId) =>
+        actorRef ! PoisonPill
+        context.system.eventStream.publish(ProjectClosed(projectId))
+      }
       context.stop(self)
   }
 
@@ -80,7 +90,7 @@ object LanguageServerKiller {
     * @return a configuration object
     */
   def props(
-    controllers: List[ActorRef],
+    controllers: Map[UUID, ActorRef],
     shutdownTimeout: FiniteDuration
   ): Props = Props(new LanguageServerKiller(controllers, shutdownTimeout))
 

lib/scala/project-manager/src/main/scala/org/enso/projectmanager/infrastructure/languageserver/LanguageServerRegistry.scala

@@ -88,7 +88,7 @@ class LanguageServerRegistry(
     case msg @ KillThemAll =>
       val killer = context.actorOf(
         LanguageServerKiller.props(
-          serverControllers.values.toList,
+          serverControllers,
           timeoutConfig.shutdownTimeout
         ),
         "language-server-killer"

lib/scala/project-manager/src/main/scala/org/enso/projectmanager/infrastructure/languageserver/ShutdownHookActivator.scala

@@ -32,26 +32,36 @@ class ShutdownHookActivator[F[+_, +_]: Exec: CovariantFlatMap]
 
   private def running(
     hooks: Map[UUID, List[ShutdownHook[F]]] =
-      Map.empty.withDefaultValue(List.empty)
+      Map.empty.withDefaultValue(List.empty),
+    scheduled: List[UUID] = Nil
   ): Receive = {
     case RegisterShutdownHook(projectId, hook) =>
       val realHook = hook.asInstanceOf[ShutdownHook[F]]
       val updated  = hooks.updated(projectId, realHook :: hooks(projectId))
-      context.become(running(updated))
+      context.become(running(updated, scheduled))
 
     case ProjectClosed(projectId) =>
       val projectHooks = hooks(projectId)
       if (projectHooks.nonEmpty) {
         context.actorOf(
           ShutdownHookRunner.props[F](projectId, projectHooks.reverse)
         )
+        context.become(running(hooks - projectId, projectId :: scheduled))
+      } else if (scheduled.contains(projectId)) {
+        logger.debug(
+          s"Request for starting shutdown hooks has already been filed for project ${projectId}. Ignoring."
+        )
+      } else {
+        logger.warn(
+          s"Shutdown hook activator has no recollection of project ${projectId}. Either it was closed already or it never existed. Ignoring."
+        )
       }
 
     case ShutdownHooksFired(projectId) =>
-      context.become(running(hooks - projectId))
+      context.become(running(hooks, scheduled.filter(_ != projectId)))
 
     case ArePendingShutdownHooks =>
-      val arePending = hooks.values.map(_.size).sum != 0
+      val arePending = hooks.values.map(_.size).sum != 0 || scheduled.nonEmpty
       sender() ! arePending
   }