Always log to console and file

CHANGELOG.md

@@ -962,6 +962,7 @@
 - [Support runtime checks of intersection types][7769]
 - [Merge `Small_Integer` and `Big_Integer` types][7636]
 - [Inline type ascriptions][7796]
+- [Always persist `TRACE` level logs to a file][7825]
 - [Downloadable VSCode extension][7861]
 - [New `project/status` route for reporting LS state][7801]
 
@@ -1106,6 +1107,7 @@
 [7636]: https://github.com/enso-org/enso/pull/7636
 [7796]: https://github.com/enso-org/enso/pull/7796
 [7801]: https://github.com/enso-org/enso/pull/7801
+[7825]: https://github.com/enso-org/enso/pull/7825
 [7861]: https://github.com/enso-org/enso/pull/7861
 
 # Enso 2.0.0-alpha.18 (2021-10-12)

build.sbt

@@ -882,6 +882,7 @@ lazy val `project-manager` = (project in file("lib/scala/project-manager"))
       case _                  => MergeStrategy.first
     },
     (Test / test) := (Test / test).dependsOn(`engine-runner` / assembly).value,
+    Test / javaOptions += s"-Dconfig.file=${sourceDirectory.value}/test/resources/application.conf",
     rebuildNativeImage := NativeImage
       .buildNativeImage(
         "project-manager",
@@ -1156,6 +1157,7 @@ lazy val `language-server` = (project in file("engine/language-server"))
           .map(_.data)
           .mkString(File.pathSeparator)
       Seq(
+        s"-Dconfig.file=${sourceDirectory.value}/test/resources/application.conf",
         s"-Dtruffle.class.path.append=$runtimeClasspath",
         s"-Duser.dir=${file(".").getCanonicalPath}"
       )
@@ -1758,7 +1760,9 @@ lazy val launcher = project
     (Test / testOnly) := (Test / testOnly)
       .dependsOn(buildNativeImage)
       .dependsOn(LauncherShimsForTest.prepare())
-      .evaluated
+      .evaluated,
+    Test / fork := true,
+    Test / javaOptions += s"-Dconfig.file=${sourceDirectory.value}/test/resources/application.conf"
   )
   .dependsOn(cli)
   .dependsOn(`runtime-version-manager`)
@@ -2194,12 +2198,13 @@ lazy val `std-table` = project
       (Antlr4 / sourceManaged).value / "main" / "antlr4"
     },
     libraryDependencies ++= Seq(
-      "org.graalvm.sdk"     % "graal-sdk"               % graalMavenPackagesVersion % "provided",
-      "org.netbeans.api"    % "org-openide-util-lookup" % netbeansApiVersion        % "provided",
-      "com.univocity"       % "univocity-parsers"       % univocityParsersVersion,
-      "org.apache.poi"      % "poi-ooxml"               % poiOoxmlVersion,
-      "org.apache.xmlbeans" % "xmlbeans"                % xmlbeansVersion,
-      "org.antlr"           % "antlr4-runtime"          % antlrVersion
+      "org.graalvm.sdk"          % "graal-sdk"               % graalMavenPackagesVersion % "provided",
+      "org.netbeans.api"         % "org-openide-util-lookup" % netbeansApiVersion        % "provided",
+      "com.univocity"            % "univocity-parsers"       % univocityParsersVersion,
+      "org.apache.poi"           % "poi-ooxml"               % poiOoxmlVersion,
+      "org.apache.xmlbeans"      % "xmlbeans"                % xmlbeansVersion,
+      "org.antlr"                % "antlr4-runtime"          % antlrVersion,
+      "org.apache.logging.log4j" % "log4j-to-slf4j"          % "2.18.0" // org.apache.poi uses log4j
     ),
     Compile / packageBin := Def.task {
       val result = (Compile / packageBin).value

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/NOTICE

@@ -51,6 +51,11 @@ The license information can be found along with the copyright notices.
 Copyright notices related to this dependency can be found in the directory `org.apache.logging.log4j.log4j-api-2.18.0`.
 
 
+'log4j-to-slf4j', licensed under the Apache License, Version 2.0, is distributed with the Table.
+The license file can be found at `licenses/APACHE2.0`.
+Copyright notices related to this dependency can be found in the directory `org.apache.logging.log4j.log4j-to-slf4j-2.18.0`.
+
+
 'poi', licensed under the Apache License, Version 2.0, is distributed with the Table.
 The license information can be found along with the copyright notices.
 Copyright notices related to this dependency can be found in the directory `org.apache.poi.poi-5.2.3`.
@@ -70,3 +75,8 @@ Copyright notices related to this dependency can be found in the directory `org.
 The license file can be found at `licenses/APACHE2.0`.
 Copyright notices related to this dependency can be found in the directory `org.apache.xmlbeans.xmlbeans-5.1.1`.
 
+
+'slf4j-api', licensed under the MIT License, is distributed with the Table.
+The license file can be found at `licenses/MIT`.
+Copyright notices related to this dependency can be found in the directory `org.slf4j.slf4j-api-1.7.36`.
+

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/licenses/MIT

@@ -0,0 +1,6 @@
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/org.apache.logging.log4j.log4j-to-slf4j-2.18.0/NOTICE

@@ -0,0 +1,8 @@
+
+Apache Log4j to SLF4J Adapter
+Copyright 1999-2022 The Apache Software Foundation
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+
+

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/org.apache.logging.log4j.log4j-to-slf4j-2.18.0/NOTICES

@@ -0,0 +1 @@
+this work for additional information regarding copyright ownership.

distribution/lib/Standard/Table/0.0.0-dev/THIRD-PARTY/org.slf4j.slf4j-api-1.7.36/NOTICES

@@ -0,0 +1 @@
+Copyright (c) 2004-2011 QOS.ch

docs/infrastructure/logging.md

@@ -29,6 +29,7 @@ involving the centralized logging service.
   - [Setting Up Logging](#setting-up-logging)
   - [Log Masking](#log-masking)
   - [Logging in Tests](#logging-in-tests)
+  - [Logging to file](#logging-to-file)
 
 <!-- /MarkdownTOC -->
 
@@ -346,3 +347,15 @@ information even if the object implements custom interface for masked logging.
 The Logging Service provides a helper function `TestLogger.gatherLogs` that will
 execute the closure and collect all logs reported in the specified class. That
 way it can verify that all logs are being reported within the provided code.
+
+### Logging to file
+
+By default Enso will attempt to persist (verbose) logs into a designated log
+file. This means that even though a user might be shown `WARNING` level logs in
+the console, Logs with up to `TRACE` level will be dumped into the log file. A
+user can disable this parallel logging to a file by setting the environment
+variable:
+
+```
+ENSO_LOG_TO_FILE=false project-manager ...
+```

engine/language-server/src/main/resources/application.conf

@@ -43,4 +43,6 @@ logging-service {
   ]
   default-appender = socket
   default-appender = ${?ENSO_APPENDER_DEFAULT}
+  always-log-to-file = false
+  always-log-to-file = ${?ENSO_LOG_TO_FILE}
 }

engine/launcher/src/main/resources/META-INF/native-image/org/enso/launcher/native-image.properties

@@ -0,0 +1 @@
+Args=--initialize-at-run-time=com.typesafe.config.impl.ConfigImpl$EnvVariablesHolder,com.typesafe.config.impl.ConfigImpl$SystemPropertiesHolder

engine/launcher/src/main/resources/application.conf

@@ -30,6 +30,8 @@ logging-service {
         pattern = "[%level{lowercase=true}] [%d{yyyy-MM-dd'T'HH:mm:ssXXX}] [%logger] %msg%n%nopex"
       }
   ]
-  default-appender = file
+  default-appender = console
   default-appender = ${?ENSO_APPENDER_DEFAULT}
+  always-log-to-file = true
+  always-log-to-file = ${?ENSO_LOG_TO_FILE}
 }

engine/launcher/src/test/scala/org/enso/launcher/NativeLauncherSpec.scala

@@ -6,7 +6,10 @@ import io.circe.parser
 class NativeLauncherSpec extends NativeTest {
   "native launcher" should {
     "display its version" in {
-      val run = runLauncher(Seq("version", "--json", "--only-launcher"))
+      val run = runLauncher(
+        Seq("version", "--json", "--only-launcher"),
+        extraJVMProps = Map("ENSO_LOG_TO_FILE" -> "false")
+      )
       run should returnSuccess
 
       val version = parser.parse(run.stdout).getOrElse {

engine/launcher/src/test/scala/org/enso/launcher/NativeTest.scala

@@ -51,10 +51,12 @@ trait NativeTest
     * @param args arguments to forward to the launcher
     * @param extraEnv environment variables to override for the launched
     *                 program, do not use these to override PATH
+    * @param extraJVMProps JVM properties to append to the launcher command
     */
   def runLauncher(
     args: Seq[String],
-    extraEnv: Map[String, String] = Map.empty
+    extraEnv: Map[String, String]      = Map.empty,
+    extraJVMProps: Map[String, String] = Map.empty
   ): RunResult = {
     if (extraEnv.contains("PATH")) {
       throw new IllegalArgumentException(
@@ -64,7 +66,8 @@ trait NativeTest
 
     runCommand(
       Seq(baseLauncherLocation.toAbsolutePath.toString) ++ args,
-      extraEnv.toSeq
+      extraEnv.toSeq,
+      extraJVMProps.toSeq
     )
   }
 
@@ -75,11 +78,13 @@ trait NativeTest
     * @param args arguments to forward to the launcher
     * @param extraEnv environment variables to override for the launched
     *                 program, do not use these to override PATH
+    * @param extraJVMProps JVM properties to append to the launcher command
     */
   def runLauncherAt(
     pathToLauncher: Path,
     args: Seq[String],
-    extraEnv: Map[String, String] = Map.empty
+    extraEnv: Map[String, String]      = Map.empty,
+    extraJVMProps: Map[String, String] = Map.empty
   ): RunResult = {
     if (extraEnv.contains("PATH")) {
       throw new IllegalArgumentException(
@@ -89,16 +94,20 @@ trait NativeTest
 
     runCommand(
       Seq(pathToLauncher.toAbsolutePath.toString) ++ args,
-      extraEnv.toSeq
+      extraEnv.toSeq,
+      extraJVMProps.toSeq
     )
   }
 
+  /** Returns a relative path to the root directory of the project. */
+  def rootDirectory: Path = Path.of("../../")
+
   /** Returns the expected location of the launcher binary compiled by the
     * Native Image. This binary can be copied into various places to test its
     * functionality.
     */
   def baseLauncherLocation: Path =
-    Path.of(".").resolve(OS.executableName("enso"))
+    rootDirectory.resolve(OS.executableName("enso"))
 
   /** Creates a copy of the tested launcher binary at the specified location.
     *
@@ -124,14 +133,17 @@ trait NativeTest
     * @param args arguments to forward to the launcher
     * @param pathOverride the system PATH that should be set for the launched
     *                     program
+    * @param extraJVMProps JVM properties to append to the launcher command
     */
   def runLauncherWithPath(
     args: Seq[String],
-    pathOverride: String
+    pathOverride: String,
+    extraJVMProps: Map[String, String] = Map.empty
   ): RunResult = {
     runCommand(
       Seq(baseLauncherLocation.toAbsolutePath.toString) ++ args,
-      Seq(NativeTest.PATH -> pathOverride)
+      Seq(NativeTest.PATH -> pathOverride),
+      extraJVMProps.toSeq
     )
   }
 }

engine/launcher/src/test/scala/org/enso/launcher/PluginManagerSpec.scala

@@ -13,6 +13,8 @@ class PluginManagerSpec
     with OptionValues
     with WithTemporaryDirectory {
 
+  val extraJVMProps = Map("ENSO_LOG_TO_FILE" -> "false")
+
   def makePluginCode(name: String): Seq[String] =
     Seq(s"""echo Plugin $name.""")
 
@@ -45,7 +47,7 @@ class PluginManagerSpec
       writePlugin(path, "plugin2")
       writePlugin(path, "plugin3", prefixed = false)
 
-      val run = runLauncherWithPath(Seq("help"), path.toString)
+      val run = runLauncherWithPath(Seq("help"), path.toString, extraJVMProps)
       run should returnSuccess
       run.stdout should include("Plugin plugin1.")
       run.stdout should include("Plugin plugin2.")
@@ -56,15 +58,17 @@ class PluginManagerSpec
       val path = getTestDirectory.toAbsolutePath
       writePlugin(path, "plugin1")
 
-      val run = runLauncherWithPath(Seq("plugin1"), path.toString)
+      val run =
+        runLauncherWithPath(Seq("plugin1"), path.toString, extraJVMProps)
       run should returnSuccess
       run.stdout.trim shouldEqual "Plugin plugin1."
     }
 
     "suggest similar plugin name on typo" in {
       val path = getTestDirectory.toAbsolutePath
       writePlugin(path, "plugin1")
-      val run = runLauncherWithPath(Seq("plugin2"), path.toString)
+      val run =
+        runLauncherWithPath(Seq("plugin2"), path.toString, extraJVMProps)
       run.exitCode should not equal 0
       run.stdout should include("plugin1")
     }

engine/launcher/src/test/scala/org/enso/launcher/installation/UninstallerSpec.scala

@@ -11,6 +11,8 @@ import org.enso.testkit.WithTemporaryDirectory
 class UninstallerSpec extends NativeTest with WithTemporaryDirectory {
   def installedRoot: Path = getTestDirectory / "installed"
 
+  private val extraJVMProps = Map("ENSO_LOG_TO_FILE" -> "false")
+
   /** Prepares an installed distribution for the purposes of testing
     * uninstallation.
     *
@@ -59,7 +61,8 @@ class UninstallerSpec extends NativeTest with WithTemporaryDirectory {
       runLauncherAt(
         launcher,
         Seq("--auto-confirm", "uninstall", "distribution"),
-        env
+        env,
+        extraJVMProps
       ) should returnSuccess
 
       assert(Files.notExists(installedRoot), "Should remove the data root.")
@@ -77,7 +80,8 @@ class UninstallerSpec extends NativeTest with WithTemporaryDirectory {
       runLauncherAt(
         launcher,
         Seq("--auto-confirm", "uninstall", "distribution"),
-        env
+        env,
+        extraJVMProps
       ) should returnSuccess
 
       assert(Files.notExists(installedRoot), "Should remove the data root.")
@@ -93,7 +97,8 @@ class UninstallerSpec extends NativeTest with WithTemporaryDirectory {
       runLauncherAt(
         launcher,
         Seq("--auto-confirm", "uninstall", "distribution"),
-        env
+        env,
+        extraJVMProps
       ) should returnSuccess
 
       assert(

engine/launcher/src/test/scala/org/enso/launcher/upgrade/UpgradeSpec.scala

@@ -138,11 +138,13 @@ class UpgradeSpec
     *
     * @param args arguments for the launcher
     * @param extraEnv environment variable overrides
+    * @param extraJVMProps JVM properties to append to the launcher command
     * @return wrapped process
     */
   def startLauncher(
     args: Seq[String],
-    extraEnv: Map[String, String] = Map.empty
+    extraEnv: Map[String, String]      = Map.empty,
+    extraJVMProps: Map[String, String] = Map.empty
   ): WrappedProcess = {
     val testArgs = Seq(
       "--internal-emulate-repository",
@@ -154,7 +156,8 @@ class UpgradeSpec
       extraEnv.updated("ENSO_LAUNCHER_LOCATION", realLauncherLocation.toString)
     start(
       Seq(launcherPath.toAbsolutePath.toString) ++ testArgs ++ args,
-      env.toSeq
+      env.toSeq,
+      extraJVMProps.toSeq
     )
   }
 

engine/runner/src/main/scala/org/enso/runner/Main.scala

@@ -1024,7 +1024,7 @@ object Main {
 
   /** Default log level to use if the LOG_LEVEL option is not provided.
     */
-  val defaultLogLevel: Level = Level.ERROR
+  val defaultLogLevel: Level = Level.WARN
 
   /** Main entry point for the CLI program.
     *

engine/runtime/src/main/scala/org/enso/compiler/SerializationManager.scala

@@ -546,7 +546,7 @@ final class SerializationManager(compiler: Compiler) {
       pool.shutdownNow()
       Thread.sleep(100)
       compiler.context.logSerializationManager(
-        debugLogLevel,
+        Level.WARNING,
         "Serialization manager has been shut down."
       )
     }

lib/scala/logging-config/src/main/java/org/enso/logger/config/Appender.java

@@ -66,10 +66,6 @@ public boolean setupForPath(
     return setup(logLevel, loggerSetup);
   }
 
-  public boolean setupForURI(Level logLevel, String hostname, int port, LoggerSetup loggerSetup) {
-    return setup(logLevel, loggerSetup);
-  }
-
   public static final String defaultPattern =
       "[%level] [%d{yyyy-MM-dd'T'HH:mm:ssXXX}] [%logger] %msg%n";
   protected static final String patternKey = "pattern";

lib/scala/logging-config/src/main/java/org/enso/logger/config/BaseConfig.java

@@ -0,0 +1,25 @@
+package org.enso.logger.config;
+
+import java.util.Map;
+
+/** Base config corresponding to the main logger section in the application config. */
+public interface BaseConfig {
+
+  /** Returns the default appender. */
+  Appender getAppender();
+
+  /** Returns a map of appenders defined in the logger section of the config. */
+  Map<String, Appender> getAppenders();
+
+  /**
+   * Returns true, if logging infrastructure should always log in verbose mode, irrespective of the
+   * log target.
+   */
+  boolean logToFile();
+
+  /**
+   * Returns a list of custom loggers and their levels that need to be taken into account when
+   * logging events.
+   */
+  LoggersLevels getLoggers();
+}