From fe6e649eca99bd2df1152f9c539819d3cfc16079 Mon Sep 17 00:00:00 2001 From: David Pilato Date: Mon, 9 Nov 2020 17:06:20 +0100 Subject: [PATCH] Split console logs and actual logs Simplify `log4j2.xml` for the end user. Now you can control where FSCrawler will store the logs and the log levels by setting `LOG_DIR`, `LOG_LEVEL` and `DOC_LEVEL` Java properties. ```sh FS_JAVA_OPTS="-DLOG_DIR=path/to/logs_dir -DLOG_LEVEL=trace -DDOC_LEVEL=debug" bin/fscrawler ``` In the future we might remove the `--debug` and `--trace` CLI options. Also adds a banner! :) Follow up for #1031 --- .gitignore | 1 + .../crawler/fs/cli/FsCrawlerCli.java | 116 ++++++++++++++---- cli/src/main/resources/log4j2.xml | 33 +++-- docs/source/admin/cli-options.rst | 6 +- docs/source/admin/logger.rst | 21 +++- .../crawler/fs/framework/FSCrawlerLogger.java | 21 ++++ 6 files changed, 156 insertions(+), 42 deletions(-) diff --git a/.gitignore b/.gitignore index 4db33f382..50e0adaa3 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ target /.project .idea *.iml +/logs/ diff --git a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java index e48ea940b..d333bdc1f 100644 --- a/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java +++ b/cli/src/main/java/fr/pilato/elasticsearch/crawler/fs/cli/FsCrawlerCli.java @@ -23,8 +23,10 @@ import com.beust.jcommander.Parameter; import fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl; import fr.pilato.elasticsearch.crawler.fs.beans.FsJobFileHandler; +import fr.pilato.elasticsearch.crawler.fs.framework.FSCrawlerLogger; import fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil; import fr.pilato.elasticsearch.crawler.fs.framework.MetaFileHandler; +import fr.pilato.elasticsearch.crawler.fs.framework.Version; import fr.pilato.elasticsearch.crawler.fs.rest.RestServer; import fr.pilato.elasticsearch.crawler.fs.settings.Elasticsearch; import fr.pilato.elasticsearch.crawler.fs.settings.Fs; @@ -32,12 +34,17 @@ import fr.pilato.elasticsearch.crawler.fs.settings.FsSettings; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsFileHandler; import fr.pilato.elasticsearch.crawler.fs.settings.FsSettingsParser; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.core.Filter; import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.filter.LevelMatchFilter; +import org.apache.logging.log4j.core.filter.LevelRangeFilter; import java.io.IOException; import java.nio.file.NoSuchFileException; @@ -46,9 +53,7 @@ import java.util.List; import java.util.Scanner; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.copyDefaultResources; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.extractMajorVersion; -import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.readDefaultJsonVersionedFile; +import static fr.pilato.elasticsearch.crawler.fs.framework.FsCrawlerUtil.*; /** * Main entry point to launch FsCrawler @@ -110,29 +115,33 @@ public static void main(String[] args) throws Exception { if (commands.debug || commands.trace || commands.silent) { LoggerContext ctx = (LoggerContext) LogManager.getContext(false); Configuration config = ctx.getConfiguration(); - LoggerConfig loggerConfig = config.getLoggerConfig(FsCrawlerCli.class.getPackage().getName()); + LoggerConfig loggerConfig = config.getLoggerConfig("fr.pilato.elasticsearch.crawler.fs"); + ConsoleAppender console = config.getAppender("Console"); if (commands.silent) { - // Check if the user also asked for --debug or --trace which is contradictory - if (commands.debug || commands.trace) { - logger.warn("--debug or --trace can't be used when --silent is set. Only silent mode will be activated."); - } // If the user did not enter any job name, nothing will be displayed if (commands.jobName == null) { + banner(); logger.warn("--silent is set but no job has been defined. Add a job name or remove --silent option. Exiting."); jCommander.usage(); return; } - // We change the full rootLogger level - LoggerConfig rootLogger = config.getLoggerConfig(LogManager.ROOT_LOGGER_NAME); - loggerConfig.setLevel(Level.OFF); - rootLogger.setLevel(Level.OFF); + // We don't write anything on the console anymore + console.addFilter(LevelMatchFilter.newBuilder().setLevel(Level.ALL).setOnMatch(Filter.Result.DENY).build()); } else { - loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE); + console.addFilter(LevelRangeFilter.createFilter( + commands.debug ? Level.DEBUG : Level.TRACE, + Level.ALL, + Filter.Result.DENY, + Filter.Result.ACCEPT)); } + + loggerConfig.setLevel(commands.debug ? Level.DEBUG : Level.TRACE); ctx.updateLoggers(); } + banner(); + if (commands.help) { jCommander.usage(); return; @@ -162,23 +171,23 @@ public static void main(String[] args) throws Exception { if (commands.jobName == null) { // The user did not enter a job name. // We can list available jobs for him - logger.info("No job specified. Here is the list of existing jobs:"); + FSCrawlerLogger.console("No job specified. Here is the list of existing jobs:"); List files = FsCrawlerJobsUtil.listExistingJobs(configDir); if (!files.isEmpty()) { for (int i = 0; i < files.size(); i++) { - logger.info("[{}] - {}", i+1, files.get(i)); + FSCrawlerLogger.console("[{}] - {}", i+1, files.get(i)); } int chosenFile = 0; while (chosenFile <= 0 || chosenFile > files.size()) { - logger.info("Choose your job [1-{}]...", files.size()); + FSCrawlerLogger.console("Choose your job [1-{}]...", files.size()); chosenFile = scanner.nextInt(); } jobName = files.get(chosenFile - 1); } else { - logger.info("No job exists in [{}].", configDir); - logger.info("To create your first job, run 'fscrawler job_name' with 'job_name' you want"); + FSCrawlerLogger.console("No job exists in [{}].", configDir); + FSCrawlerLogger.console("To create your first job, run 'fscrawler job_name' with 'job_name' you want"); return; } @@ -210,18 +219,24 @@ public static void main(String[] args) throws Exception { } if (username != null && fsSettings.getElasticsearch().getPassword() == null) { - logger.info("Password for " + username + ":"); + FSCrawlerLogger.console("Password for {}:", username); String password = scanner.next(); fsSettings.getElasticsearch().setUsername(username); fsSettings.getElasticsearch().setPassword(password); } } catch (NoSuchFileException e) { - logger.warn("job [{}] does not exist", jobName); + // We can only have a dialog with the end user if we are not silent + if (commands.silent) { + logger.error("job [{}] does not exist. Exiting as we are in silent mode.", jobName); + return; + } + + FSCrawlerLogger.console("job [{}] does not exist", jobName); String yesno = null; while (!"y".equalsIgnoreCase(yesno) && !"n".equalsIgnoreCase(yesno)) { - logger.info("Do you want to create it (Y/N)?"); + FSCrawlerLogger.console("Do you want to create it (Y/N)?"); yesno = scanner.next(); } @@ -233,7 +248,7 @@ public static void main(String[] args) throws Exception { fsSettingsFileHandler.write(fsSettings); Path config = configDir.resolve(jobName).resolve(FsSettingsFileHandler.SETTINGS_YAML); - logger.info("Settings have been created in [{}]. Please review and edit before relaunch", config); + FSCrawlerLogger.console("Settings have been created in [{}]. Please review and edit before relaunch", config); } return; @@ -281,6 +296,63 @@ public static void main(String[] args) throws Exception { } } + private final static int bannerLength = 100; + + /** + * This is coming from: https://patorjk.com/software/taag/#p=display&f=3D%20Diagonal&t=FSCrawler + */ + private final static String asciiArt = "" + + " ,---,. .--.--. ,----.. ,--, \n" + + " ,' .' | / / '. / / \\ ,--.'| \n" + + ",---.' || : /`. / | : : __ ,-. .---.| | : __ ,-.\n" + + "| | .'; | |--` . | ;. /,' ,'/ /| /. ./|: : ' ,' ,'/ /|\n" + + ": : : | : ;_ . ; /--` ' | |' | ,--.--. .-'-. ' || ' | ,---. ' | |' |\n" + + ": | |-, \\ \\ `. ; | ; | | ,'/ \\ /___/ \\: |' | | / \\ | | ,'\n" + + "| : ;/| `----. \\| : | ' : / .--. .-. | .-'.. ' ' .| | : / / |' : / \n" + + "| | .' __ \\ \\ |. | '___ | | ' \\__\\/: . ./___/ \\: '' : |__ . ' / || | ' \n" + + "' : ' / /`--' /' ; : .'|; : | ,\" .--.; |. \\ ' .\\ | | '.'|' ; /|; : | \n" + + "| | | '--'. / ' | '/ :| , ; / / ,. | \\ \\ ' \\ |; : ;' | / || , ; \n" + + "| : \\ `--'---' | : / ---' ; : .' \\ \\ \\ |--\" | , / | : | ---' \n" + + "| | ,' \\ \\ .' | , .-./ \\ \\ | ---`-' \\ \\ / \n" + + "`----' `---` `--`---' '---\" `----' \n"; + + private static void banner() { + FSCrawlerLogger.console( + separatorLine(",", ".") + + centerAsciiArt() + + separatorLine("+", "+") + + bannerLine("You know, for Files!") + + bannerLine("Made from France with Love") + + bannerLine("Source: https://github.com/dadoonet/fscrawler/") + + bannerLine("Documentation: https://fscrawler.readthedocs.io/") + + separatorLine("`", "'")); + } + + private static String centerAsciiArt() { + String[] lines = StringUtils.split(asciiArt, '\n'); + + // Edit line 0 as we want to add the version + String version = Version.getVersion(); + String firstLine = StringUtils.stripEnd(StringUtils.center(lines[0], bannerLength), null); + String pad = StringUtils.rightPad(firstLine, bannerLength - version.length() - 1) + version; + lines[0] = pad; + + StringBuilder content = new StringBuilder(); + for (String line : lines) { + content.append(bannerLine(line)); + } + + return content.toString(); + } + + private static String bannerLine(String text) { + return "|" + StringUtils.center(text, bannerLength) + "|\n"; + } + + private static String separatorLine(String first, String last) { + return first + StringUtils.center("", bannerLength, "-") + last + "\n"; + } + private static void checkForDeprecatedResources(Path configDir, String elasticsearchVersion) throws IOException { try { // If we are able to read an old configuration file, we should tell the user to check the documentation diff --git a/cli/src/main/resources/log4j2.xml b/cli/src/main/resources/log4j2.xml index 59dd21eb2..6072531c0 100644 --- a/cli/src/main/resources/log4j2.xml +++ b/cli/src/main/resources/log4j2.xml @@ -1,19 +1,22 @@ - %d{ABSOLUTE} %highlight{%-5p} [%c{1.}] %m%n - %d [%highlight{%-5p}] %m%n + + info + + info + logs - - + + - + @@ -24,7 +27,7 @@ - + @@ -32,29 +35,33 @@ + + + + + - + - - + - + - + - + - + diff --git a/docs/source/admin/cli-options.rst b/docs/source/admin/cli-options.rst index 540dc7d71..0bd67ba06 100644 --- a/docs/source/admin/cli-options.rst +++ b/docs/source/admin/cli-options.rst @@ -4,9 +4,9 @@ CLI options =========== - ``--help`` displays help -- ``--silent`` runs in silent mode. No output is generated. -- ``--debug`` runs in debug mode. -- ``--trace`` runs in trace mode (more verbose than debug). +- ``--silent`` runs in silent mode. No output is generated on the console. +- ``--debug`` runs in debug mode. This applies to log files only. See also :ref:`logger`. +- ``--trace`` runs in trace mode (more verbose than debug). This applies to log files only. See also :ref:`logger`. - ``--config_dir`` defines directory where jobs are stored instead of default ``~/.fscrawler``. - ``--username`` defines the username to use when using an secured diff --git a/docs/source/admin/logger.rst b/docs/source/admin/logger.rst index a00398f68..a5883df0f 100644 --- a/docs/source/admin/logger.rst +++ b/docs/source/admin/logger.rst @@ -1,16 +1,29 @@ +.. _logger: + Configuring the logger ====================== -FSCrawler comes with a default logger configuration which can be found in the +In addition to the :ref:`cli-options`, FSCrawler comes with a default logger configuration which can be found in the FSCrawler installation dir as ``config/log4j2.xml`` file. -You can modify it to suit your needs. +You can modify it to suit your needs. It will be automatically reloaded every 30 seconds. + +There are some properties to make your life easier to change the log levels or the log dir: + +.. code:: xml + + + info + info + logs + -You can control where FSCrawler will store the logs by setting the ``LOG_DIR`` Java property. +You can control where FSCrawler will store the logs and the log levels by setting +``LOG_DIR``, ``LOG_LEVEL`` and ``DOC_LEVEL`` Java properties. .. code:: sh - FS_JAVA_OPTS="-DLOG_DIR=path/to/logs_dir" bin/fscrawler + FS_JAVA_OPTS="-DLOG_DIR=path/to/logs_dir -DLOG_LEVEL=trace -DDOC_LEVEL=debug" bin/fscrawler By default, it will log everything in the ``logs`` directory inside the installation folder. diff --git a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java index d54edf9f8..035a00c82 100644 --- a/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java +++ b/framework/src/main/java/fr/pilato/elasticsearch/crawler/fs/framework/FSCrawlerLogger.java @@ -24,15 +24,36 @@ public class FSCrawlerLogger { + /** + * This logger is for the console + */ + private final static Logger consoleLogger = LogManager.getLogger("fscrawler.console"); + /** * This logger is used to log information related to documents */ private final static Logger documentLogger = LogManager.getLogger("fscrawler.document"); + public static void console(String message, Object... params) { + consoleLogger.info(message, params); + } + + /** + * Log information in Debug Level about documents + * @param id Document ID + * @param path Virtual path to the document + * @param message Message to display + */ public static void documentDebug(String id, String path, String message) { documentLogger.debug("[{}][{}] {}", id, path, message); } + /** + * Log information in Error Level about documents + * @param id Document ID + * @param path Virtual path to the document + * @param error Error to display + */ public static void documentError(String id, String path, String error) { documentLogger.error("[{}][{}] {}", id, path, error); }