Merge pull request #1037 from dadoonet/pr/log-index-errors

Split console logs and actual logs and add a banner :)
dadoonet · Nov 13, 2020 · 1d84d9f · 1d84d9f
2 parents b52128c + fe6e649
commit 1d84d9f
Show file tree

Hide file tree

Showing 6 changed files with 156 additions and 42 deletions.
diff --git a/.gitignore b/.gitignore
@@ -4,3 +4,4 @@ target
 /.project
 .idea
 *.iml
+/logs/
diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java
@@ -23,21 +23,28 @@
 import com.beust.jcommander.Parameter;
 import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl;
 import fr.pilato.elasticsearch.crawler.fs.beans.FsJobFileHandler;
+import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger;
 import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil;
 import fr.pilato.elasticsearch.crawler.fs.framework.MetaFileHandler;
+import fr.pilato.elasticsearch.crawler.fs.framework.Version;
 import fr.pilato.elasticsearch.crawler.fs.rest.RestServer;
 import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch;
 import fr.pilato.elasticsearch.crawler.fs.settings.Fs;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsCrawlerValidator;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler;
 import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsParser;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.apache.logging.log4j.core.Filter;
 import org.apache.logging.log4j.core.LoggerContext;
+import org.apache.logging.log4j.core.appender.ConsoleAppender;
 import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.LoggerConfig;
+import org.apache.logging.log4j.core.filter.LevelMatchFilter;
+import org.apache.logging.log4j.core.filter.LevelRangeFilter;
 
 import java.io.IOException;
 import java.nio.file.NoSuchFileException;
@@ -46,9 +53,7 @@
 import java.util.List;
 import java.util.Scanner;
 
-import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources;
-import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion;
-import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.readDefaultJsonVersionedFile;
+import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*;
 
 /**
  * Main entry point to launch FsCrawler
@@ -110,29 +115,33 @@ public static void main(String[] args) throws Exception {
         if (commands.debug || commands.trace || commands.silent) {
             LoggerContext ctx = (LoggerContext) LogManager.getContext(false);
             Configuration config = ctx.getConfiguration();
-            LoggerConfig loggerConfig = config.getLoggerConfig(FsCrawlerCli.class.getPackage().getName());
+            LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs");
+            ConsoleAppender console = config.getAppender("Console");
 
             if (commands.silent) {
-                // Check if the user also asked for --debug or --trace which is contradictory
-                if (commands.debug || commands.trace) {
-                    logger.warn("--debug or --trace can't be used when --silent is set. Only silent mode will be activated.");
-                }
                 // If the user did not enter any job name, nothing will be displayed
                 if (commands.jobName == null) {
+                    banner();
                     logger.warn("--silent is set but no job has been defined. Add a job name or remove --silent option. Exiting.");
                     jCommander.usage();
                     return;
                 }
-                // We change the full rootLogger level
-                LoggerConfig rootLogger = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME);
-                loggerConfig.setLevel(Level.OFF);
-                rootLogger.setLevel(Level.OFF);
+                // We don't write anything on the console anymore
+                console.addFilter(LevelMatchFilter.newBuilder().setLevel(Level.ALL).setOnMatch(Filter.Result.DENY).build());
             } else {
-                loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE);
+                console.addFilter(LevelRangeFilter.createFilter(
+                        commands.debug ? Level.DEBUG : Level.TRACE,
+                        Level.ALL,
+                        Filter.Result.DENY,
+                        Filter.Result.ACCEPT));
             }
+
+            loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE);
             ctx.updateLoggers();
         }
 
+        banner();
+
         if (commands.help) {
             jCommander.usage();
             return;
@@ -162,23 +171,23 @@ public static void main(String[] args) throws Exception {
         if (commands.jobName == null) {
             // The user did not enter a job name.
             // We can list available jobs for him
-            logger.info("No job specified. Here is the list of existing jobs:");
+            FSCrawlerLogger.console("No job specified. Here is the list of existing jobs:");
 
             List<String> files = FsCrawlerJobsUtil.listExistingJobs(configDir);
 
             if (!files.isEmpty()) {
                 for (int i = 0; i < files.size(); i++) {
-                    logger.info("[{}] - {}", i+1, files.get(i));
+                    FSCrawlerLogger.console("[{}] - {}", i+1, files.get(i));
                 }
                 int chosenFile = 0;
                 while (chosenFile <= 0 || chosenFile > files.size()) {
-                    logger.info("Choose your job [1-{}]...", files.size());
+                    FSCrawlerLogger.console("Choose your job [1-{}]...", files.size());
                     chosenFile = scanner.nextInt();
                 }
                 jobName = files.get(chosenFile - 1);
             } else {
-                logger.info("No job exists in [{}].", configDir);
-                logger.info("To create your first job, run 'fscrawler job_name' with 'job_name' you want");
+                FSCrawlerLogger.console("No job exists in [{}].", configDir);
+                FSCrawlerLogger.console("To create your first job, run 'fscrawler job_name' with 'job_name' you want");
                 return;
             }
 
@@ -210,18 +219,24 @@ public static void main(String[] args) throws Exception {
             }
 
             if (username != null && fsSettings.getElasticsearch().getPassword() == null) {
-                logger.info("Password for " +  username + ":");
+                FSCrawlerLogger.console("Password for {}:", username);
                 String password = scanner.next();
                 fsSettings.getElasticsearch().setUsername(username);
                 fsSettings.getElasticsearch().setPassword(password);
             }
 
         } catch (NoSuchFileException e) {
-            logger.warn("job [{}] does not exist", jobName);
+            // We can only have a dialog with the end user if we are not silent
+            if (commands.silent) {
+                logger.error("job [{}] does not exist. Exiting as we are in silent mode.", jobName);
+                return;
+            }
+
+            FSCrawlerLogger.console("job [{}] does not exist", jobName);
 
             String yesno = null;
             while (!"y".equalsIgnoreCase(yesno) && !"n".equalsIgnoreCase(yesno)) {
-                logger.info("Do you want to create it (Y/N)?");
+                FSCrawlerLogger.console("Do you want to create it (Y/N)?");
                 yesno = scanner.next();
             }
 
@@ -233,7 +248,7 @@ public static void main(String[] args) throws Exception {
                 fsSettingsFileHandler.write(fsSettings);
 
                 Path config = configDir.resolve(jobName).resolve(FsSettingsFileHandler.SETTINGS_YAML);
-                logger.info("Settings have been created in [{}]. Please review and edit before relaunch", config);
+                FSCrawlerLogger.console("Settings have been created in [{}]. Please review and edit before relaunch", config);
             }
 
             return;
@@ -281,6 +296,63 @@ public static void main(String[] args) throws Exception {
         }
     }
 
+    private final static int bannerLength = 100;
+
+    /**
+     * This is coming from: https://patorjk.com/software/taag/#p=display&f=3D%20Diagonal&t=FSCrawler
+     */
+    private final static String asciiArt = "" +
+            "    ,---,.  .--.--.     ,----..                                     ,--,                      \n" +
+            "  ,'  .' | /  /    '.  /   /   \\                                  ,--.'|                      \n" +
+            ",---.'   ||  :  /`. / |   :     :  __  ,-.                   .---.|  | :               __  ,-.\n" +
+            "|   |   .';  |  |--`  .   |  ;. /,' ,'/ /|                  /. ./|:  : '             ,' ,'/ /|\n" +
+            ":   :  :  |  :  ;_    .   ; /--` '  | |' | ,--.--.       .-'-. ' ||  ' |      ,---.  '  | |' |\n" +
+            ":   |  |-, \\  \\    `. ;   | ;    |  |   ,'/       \\     /___/ \\: |'  | |     /     \\ |  |   ,'\n" +
+            "|   :  ;/|  `----.   \\|   : |    '  :  / .--.  .-. | .-'.. '   ' .|  | :    /    /  |'  :  /  \n" +
+            "|   |   .'  __ \\  \\  |.   | '___ |  | '   \\__\\/: . ./___/ \\:     ''  : |__ .    ' / ||  | '   \n" +
+            "'   :  '   /  /`--'  /'   ; : .'|;  : |   ,\" .--.; |.   \\  ' .\\   |  | '.'|'   ;   /|;  : |   \n" +
+            "|   |  |  '--'.     / '   | '/  :|  , ;  /  /  ,.  | \\   \\   ' \\ |;  :    ;'   |  / ||  , ;   \n" +
+            "|   :  \\    `--'---'  |   :    /  ---'  ;  :   .'   \\ \\   \\  |--\" |  ,   / |   :    | ---'    \n" +
+            "|   | ,'               \\   \\ .'         |  ,     .-./  \\   \\ |     ---`-'   \\   \\  /          \n" +
+            "`----'                  `---`            `--`---'       '---\"                `----'           \n";
+
+    private static void banner() {
+        FSCrawlerLogger.console(
+                separatorLine(",", ".") +
+                centerAsciiArt() +
+                separatorLine("+", "+") +
+                bannerLine("You know, for Files!") +
+                bannerLine("Made from France with Love") +
+                bannerLine("Source: https://github.com/dadoonet/fscrawler/") +
+                bannerLine("Documentation: https://fscrawler.readthedocs.io/") +
+                separatorLine("`", "'"));
+    }
+
+    private static String centerAsciiArt() {
+        String[] lines = StringUtils.split(asciiArt, '\n');
+
+        // Edit line 0 as we want to add the version
+        String version = Version.getVersion();
+        String firstLine = StringUtils.stripEnd(StringUtils.center(lines[0], bannerLength), null);
+        String pad = StringUtils.rightPad(firstLine, bannerLength - version.length() - 1) + version;
+        lines[0] = pad;
+
+        StringBuilder content = new StringBuilder();
+        for (String line : lines) {
+            content.append(bannerLine(line));
+        }
+
+        return content.toString();
+    }
+
+    private static String bannerLine(String text) {
+        return "|" + StringUtils.center(text, bannerLength) + "|\n";
+    }
+
+    private static String separatorLine(String first, String last) {
+        return first + StringUtils.center("", bannerLength, "-") + last + "\n";
+    }
+
     private static void checkForDeprecatedResources(Path configDir, String elasticsearchVersion) throws IOException {
         try {
             // If we are able to read an old configuration file, we should tell the user to check the documentation

diff --git a/cli/src/main/resources/log4j2.xml b/cli/src/main/resources/log4j2.xml
@@ -1,19 +1,22 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <Configuration status="fatal" monitorInterval="30">
    <Properties>
-      <Property name="LOG_PATTERN">%d{ABSOLUTE} %highlight{%-5p} [%c{1.}] %m%n</Property>
-      <Property name="DOCS_PATTERN">%d [%highlight{%-5p}] %m%n</Property>
+      <!-- If you want to change the log level for fscrawler.log file -->
+      <Property name="LOG_LEVEL">info</Property>
+      <!-- If you want to change the log level for documents.log file -->
+      <Property name="DOC_LEVEL">info</Property>
+      <!-- If you want to change the output dir for logs -->
       <Property name="LOG_DIR">logs</Property>
    </Properties>
 
    <Appenders>
-      <Console name="CONSOLE" target="SYSTEM_OUT" follow="true">
-         <PatternLayout pattern="${LOG_PATTERN}"/>
+      <Console name="Console" target="SYSTEM_OUT" follow="true">
+         <PatternLayout pattern="%m%n"/>
       </Console>
 
       <RollingFile name="RollingFile" fileName="${sys:LOG_DIR}/fscrawler.log"
                    filePattern="${sys:LOG_DIR}/fscrawler-%d{yyyy-MM-dd}-%i.log.gz">
-         <PatternLayout pattern="${LOG_PATTERN}"/>
+         <PatternLayout pattern="%d{ABSOLUTE} %highlight{%-5p} [%c{1.}] %m%n"/>
          <Policies>
             <OnStartupTriggeringPolicy />
             <SizeBasedTriggeringPolicy size="20 MB" />
@@ -24,37 +27,41 @@
 
       <RollingFile name="Documents" fileName="${sys:LOG_DIR}/documents.log"
                    filePattern="${sys:LOG_DIR}/documents-%d{yyyy-MM-dd}.log.gz">
-         <PatternLayout pattern="${DOCS_PATTERN}"/>
+         <PatternLayout pattern="%d [%highlight{%-5p}] %m%n"/>
          <Policies>
             <TimeBasedTriggeringPolicy />
          </Policies>
          <DefaultRolloverStrategy max="7"/>
       </RollingFile>
    </Appenders>
    <Loggers>
+      <!-- This logger is used for the console -->
+      <Logger name="fscrawler.console" level="info" additivity="false">
+         <AppenderRef ref="Console" />
+      </Logger>
+
       <!-- This logger is used to trace all information about documents -->
-      <Logger name="fscrawler.document" level="info" additivity="false">
+      <Logger name="fscrawler.document" level="${sys:DOC_LEVEL}" additivity="false">
          <AppenderRef ref="Documents" />
       </Logger>
 
       <!-- This logger is used to log FSCrawler code execution -->
-      <Logger name="fr.pilato.elasticsearch.crawler.fs" level="info" additivity="false">
-         <AppenderRef ref="CONSOLE"/>
+      <Logger name="fr.pilato.elasticsearch.crawler.fs" level="${sys:LOG_LEVEL}" additivity="false">
          <AppenderRef ref="RollingFile" />
       </Logger>
 
       <!-- This logger is used to log 3rd party libs execution -->
       <Logger name="org.elasticsearch" level="warn" additivity="false">
-         <AppenderRef ref="CONSOLE"/>
+         <AppenderRef ref="RollingFile" />
       </Logger>
       <Logger name="org.glassfish" level="warn" additivity="false">
-         <AppenderRef ref="CONSOLE"/>
+         <AppenderRef ref="RollingFile" />
       </Logger>
       <Logger name="org.apache.tika.parser.ocr.TesseractOCRParser" level="error" additivity="false">
-         <AppenderRef ref="CONSOLE"/>
+         <AppenderRef ref="RollingFile" />
       </Logger>
 
-      <Root level="info">
+      <Root level="warn">
          <AppenderRef ref="RollingFile" />
       </Root>
    </Loggers>

diff --git a/docs/source/admin/cli-options.rst b/docs/source/admin/cli-options.rst
@@ -4,9 +4,9 @@ CLI options
 ===========
 
 -  ``--help`` displays help
--  ``--silent`` runs in silent mode. No output is generated.
--  ``--debug`` runs in debug mode.
--  ``--trace`` runs in trace mode (more verbose than debug).
+-  ``--silent`` runs in silent mode. No output is generated on the console.
+-  ``--debug`` runs in debug mode. This applies to log files only. See also :ref:`logger`.
+-  ``--trace`` runs in trace mode (more verbose than debug). This applies to log files only. See also :ref:`logger`.
 -  ``--config_dir`` defines directory where jobs are stored instead of
    default ``~/.fscrawler``.
 -  ``--username`` defines the username to use when using an secured

diff --git a/docs/source/admin/logger.rst b/docs/source/admin/logger.rst
@@ -1,16 +1,29 @@
+.. _logger:
+
 Configuring the logger
 ======================
 
-FSCrawler comes with a default logger configuration which can be found in the
+In addition to the :ref:`cli-options`, FSCrawler comes with a default logger configuration which can be found in the
 FSCrawler installation dir as ``config/log4j2.xml`` file.
 
-You can modify it to suit your needs.
+You can modify it to suit your needs. It will be automatically reloaded every 30 seconds.
+
+There are some properties to make your life easier to change the log levels or the log dir:
+
+.. code:: xml
+
+   <Properties>
+      <Property name="LOG_LEVEL">info</Property>
+      <Property name="DOC_LEVEL">info</Property>
+      <Property name="LOG_DIR">logs</Property>
+   </Properties>
 
-You can control where FSCrawler will store the logs by setting the ``LOG_DIR`` Java property.
+You can control where FSCrawler will store the logs and the log levels by setting
+``LOG_DIR``, ``LOG_LEVEL`` and ``DOC_LEVEL`` Java properties.
 
 .. code:: sh
 
-   FS_JAVA_OPTS="-DLOG_DIR=path/to/logs_dir" bin/fscrawler
+   FS_JAVA_OPTS="-DLOG_DIR=path/to/logs_dir -DLOG_LEVEL=trace -DDOC_LEVEL=debug" bin/fscrawler
 
 By default, it will log everything in the ``logs`` directory inside the installation folder.
 

diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java
@@ -24,15 +24,36 @@
 
 public class FSCrawlerLogger {
 
+    /**
+     * This logger is for the console
+     */
+    private final static Logger consoleLogger = LogManager.getLogger("fscrawler.console");
+
     /**
      * This logger is used to log information related to documents
      */
     private final static Logger documentLogger = LogManager.getLogger("fscrawler.document");
 
+    public static void console(String message, Object... params) {
+        consoleLogger.info(message, params);
+    }
+
+    /**
+     * Log information in Debug Level about documents
+     * @param id        Document ID
+     * @param path      Virtual path to the document
+     * @param message   Message to display
+     */
     public static void documentDebug(String id, String path, String message) {
         documentLogger.debug("[{}][{}] {}", id, path, message);
     }
 
+    /**
+     * Log information in Error Level about documents
+     * @param id        Document ID
+     * @param path      Virtual path to the document
+     * @param error     Error to display
+     */
     public static void documentError(String id, String path, String error) {
         documentLogger.error("[{}][{}] {}", id, path, error);
     }
-Original file line number
+Diff line change
@@ Expand Up / @@ -4,3 +4,4 @@ target @@
     /.project
     .idea
     *.iml
+    /logs/