diff --git a/buildSrc/version.properties b/buildSrc/version.properties index 36821660bce87..6c4116f8bbc8e 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -16,6 +16,7 @@ supercsv = 2.4.0 # when updating log4j, please update also docs/java-api/index.asciidoc log4j = 2.11.1 slf4j = 1.6.2 +ecsLogging = 0.1.3 # when updating the JNA version, also update the version in buildSrc/build.gradle jna = 4.5.1 diff --git a/distribution/docker/src/docker/config/log4j2.properties b/distribution/docker/src/docker/config/log4j2.properties index 129ad8bf67cd3..3594c9d2428f6 100644 --- a/distribution/docker/src/docker/config/log4j2.properties +++ b/distribution/docker/src/docker/config/log4j2.properties @@ -6,7 +6,7 @@ logger.action.level = debug appender.rolling.type = Console appender.rolling.name = rolling -appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type = ECSJsonLayout appender.rolling.layout.type_name = server rootLogger.level = info @@ -14,7 +14,7 @@ rootLogger.appenderRef.rolling.ref = rolling appender.deprecation_rolling.type = Console appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type = ECSJsonLayout appender.deprecation_rolling.layout.type_name = deprecation logger.deprecation.name = org.elasticsearch.deprecation @@ -24,9 +24,8 @@ logger.deprecation.additivity = false appender.index_search_slowlog_rolling.type = Console appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type = ECSJsonLayout appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog -appender.index_search_slowlog_rolling.layout.overrideFields=message logger.index_search_slowlog_rolling.name = index.search.slowlog logger.index_search_slowlog_rolling.level = trace @@ -35,9 +34,8 @@ logger.index_search_slowlog_rolling.additivity = false appender.index_indexing_slowlog_rolling.type = Console appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type = ECSJsonLayout appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog -appender.index_indexing_slowlog_rolling.layout.overrideFields=message logger.index_indexing_slowlog.name = index.indexing.slowlog.index logger.index_indexing_slowlog.level = trace diff --git a/distribution/docker/src/docker/config/oss/log4j2.properties b/distribution/docker/src/docker/config/oss/log4j2.properties index 73420a047edc5..596a88eb65376 100644 --- a/distribution/docker/src/docker/config/oss/log4j2.properties +++ b/distribution/docker/src/docker/config/oss/log4j2.properties @@ -6,7 +6,7 @@ logger.action.level = debug appender.rolling.type = Console appender.rolling.name = rolling -appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type = ECSJsonLayout appender.rolling.layout.type_name = server rootLogger.level = info @@ -14,7 +14,7 @@ rootLogger.appenderRef.rolling.ref = rolling appender.deprecation_rolling.type = Console appender.deprecation_rolling.name = deprecation_rolling -appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type = ECSJsonLayout appender.deprecation_rolling.layout.type_name = deprecation logger.deprecation.name = org.elasticsearch.deprecation @@ -24,7 +24,7 @@ logger.deprecation.additivity = false appender.index_search_slowlog_rolling.type = Console appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling -appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type = ECSJsonLayout appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog logger.index_search_slowlog_rolling.name = index.search.slowlog @@ -34,7 +34,7 @@ logger.index_search_slowlog_rolling.additivity = false appender.index_indexing_slowlog_rolling.type = Console appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling -appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type = ECSJsonLayout appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog logger.index_indexing_slowlog.name = index.indexing.slowlog.index diff --git a/distribution/src/config/log4j2.properties b/distribution/src/config/log4j2.properties index 681a8a2f52252..2aac6f58dc4e6 100644 --- a/distribution/src/config/log4j2.properties +++ b/distribution/src/config/log4j2.properties @@ -9,7 +9,7 @@ appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%ma appender.rolling.type = RollingFile appender.rolling.name = rolling appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json -appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type = ECSJsonLayout appender.rolling.layout.type_name = server appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz @@ -61,7 +61,7 @@ rootLogger.appenderRef.rolling_old.ref = rolling_old appender.deprecation_rolling.type = RollingFile appender.deprecation_rolling.name = deprecation_rolling appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json -appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type = ECSJsonLayout appender.deprecation_rolling.layout.type_name = deprecation appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz @@ -71,25 +71,10 @@ appender.deprecation_rolling.policies.size.size = 1GB appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy appender.deprecation_rolling.strategy.max = 4 ################################################# -######## Deprecation - old style pattern ####### -appender.deprecation_rolling_old.type = RollingFile -appender.deprecation_rolling_old.name = deprecation_rolling_old -appender.deprecation_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log -appender.deprecation_rolling_old.layout.type = PatternLayout -appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.deprecation_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _deprecation-%i.log.gz -appender.deprecation_rolling_old.policies.type = Policies -appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.deprecation_rolling_old.policies.size.size = 1GB -appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy -appender.deprecation_rolling_old.strategy.max = 4 -################################################# + logger.deprecation.name = org.elasticsearch.deprecation logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling -logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old logger.deprecation.additivity = false ######## Search slowlog JSON #################### @@ -97,9 +82,8 @@ appender.index_search_slowlog_rolling.type = RollingFile appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ .cluster_name}_index_search_slowlog.json -appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type = ECSJsonLayout appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog -appender.index_search_slowlog_rolling.layout.overrideFields=message appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ .cluster_name}_index_search_slowlog-%i.json.gz @@ -109,26 +93,11 @@ appender.index_search_slowlog_rolling.policies.size.size = 1GB appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy appender.index_search_slowlog_rolling.strategy.max = 4 ################################################# -######## Search slowlog - old style pattern #### -appender.index_search_slowlog_rolling_old.type = RollingFile -appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old -appender.index_search_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_search_slowlog.log -appender.index_search_slowlog_rolling_old.layout.type = PatternLayout -appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.index_search_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_search_slowlog-%i.log.gz -appender.index_search_slowlog_rolling_old.policies.type = Policies -appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.index_search_slowlog_rolling_old.policies.size.size = 1GB -appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.index_search_slowlog_rolling_old.strategy.max = 4 + ################################################# logger.index_search_slowlog_rolling.name = index.search.slowlog logger.index_search_slowlog_rolling.level = trace logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling -logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old logger.index_search_slowlog_rolling.additivity = false ######## Indexing slowlog JSON ################## @@ -136,9 +105,9 @@ appender.index_indexing_slowlog_rolling.type = RollingFile appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ _index_indexing_slowlog.json -appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type = ECSJsonLayout appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog -appender.index_indexing_slowlog_rolling.layout.overrideFields=message + appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ _index_indexing_slowlog-%i.json.gz @@ -148,25 +117,9 @@ appender.index_indexing_slowlog_rolling.policies.size.size = 1GB appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy appender.index_indexing_slowlog_rolling.strategy.max = 4 ################################################# -######## Indexing slowlog - old style pattern ## -appender.index_indexing_slowlog_rolling_old.type = RollingFile -appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old -appender.index_indexing_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_indexing_slowlog.log -appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout -appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ - _index_indexing_slowlog-%i.log.gz -appender.index_indexing_slowlog_rolling_old.policies.type = Policies -appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy -appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB -appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy -appender.index_indexing_slowlog_rolling_old.strategy.max = 4 -################################################# + logger.index_indexing_slowlog.name = index.indexing.slowlog.index logger.index_indexing_slowlog.level = trace logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling -logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old logger.index_indexing_slowlog.additivity = false diff --git a/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java b/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java index 29c81c1a39c0a..9844b5a709cbc 100644 --- a/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java +++ b/qa/die-with-dignity/src/main/java/org/elasticsearch/DieWithDignityPlugin.java @@ -42,13 +42,14 @@ public DieWithDignityPlugin() { @Override public List getRestHandlers( - final Settings settings, - final RestController restController, - final ClusterSettings clusterSettings, - final IndexScopedSettings indexScopedSettings, - final SettingsFilter settingsFilter, - final IndexNameExpressionResolver indexNameExpressionResolver, - final Supplier nodesInCluster) { + final Settings settings, + final RestController restController, + final ClusterSettings clusterSettings, + final IndexScopedSettings indexScopedSettings, + final SettingsFilter settingsFilter, + final IndexNameExpressionResolver indexNameExpressionResolver, + final Supplier nodesInCluster + ) { return Collections.singletonList(new RestDieWithDignityAction()); } diff --git a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java index c6350f92ae78f..a6f272dbd5940 100644 --- a/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java +++ b/qa/die-with-dignity/src/test/java/org/elasticsearch/qa/die_with_dignity/DieWithDignityIT.java @@ -39,18 +39,14 @@ public class DieWithDignityIT extends ESRestTestCase { public void testDieWithDignity() throws Exception { - expectThrows( - IOException.class, - () -> client().performRequest(new Request("GET", "/_die_with_dignity")) - ); + expectThrows(IOException.class, () -> client().performRequest(new Request("GET", "/_die_with_dignity"))); // the Elasticsearch process should die and disappear from the output of jps assertBusy(() -> { final String jpsPath = PathUtils.get(System.getProperty("runtime.java.home"), "bin/jps").toString(); final Process process = new ProcessBuilder().command(jpsPath, "-v").start(); - try (InputStream is = process.getInputStream(); - BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) { + try (InputStream is = process.getInputStream(); BufferedReader in = new BufferedReader(new InputStreamReader(is, "UTF-8"))) { String line; while ((line = in.readLine()) != null) { assertThat(line, line, not(containsString("-Ddie.with.dignity.test"))); @@ -60,7 +56,6 @@ public void testDieWithDignity() throws Exception { // parse the logs and ensure that Elasticsearch died with the expected cause final List lines = Files.readAllLines(PathUtils.get(System.getProperty("log"))); - final Iterator it = lines.iterator(); boolean fatalError = false; @@ -68,13 +63,17 @@ public void testDieWithDignity() throws Exception { try { while (it.hasNext() && (fatalError == false || fatalErrorInThreadExiting == false)) { final String line = it.next(); - if (line.matches(".*ERROR.*o\\.e\\.ExceptionsHelper.*integTest-0.*fatal error.*")) { + if (containsAll(line, ".*ERROR.*", ".*ExceptionsHelper.*", ".*integTest-0.*", ".*fatal error.*")) { fatalError = true; - } else if (line.matches(".*ERROR.*o\\.e\\.b\\.ElasticsearchUncaughtExceptionHandler.*integTest-0.*" - + "fatal error in thread \\[Thread-\\d+\\], exiting.*")) { + } else if (containsAll( + line, + ".*ERROR.*", + ".*ElasticsearchUncaughtExceptionHandler.*", + ".*integTest-0.*", + ".*fatal error in thread \\[Thread-\\d+\\], exiting.*", + ".*java.lang.OutOfMemoryError: die with dignity.*" + )) { fatalErrorInThreadExiting = true; - assertTrue(it.hasNext()); - assertThat(it.next(), containsString("java.lang.OutOfMemoryError: die with dignity")); } } @@ -88,6 +87,15 @@ public void testDieWithDignity() throws Exception { } } + private boolean containsAll(String line, String... subStrings) { + for (String subString : subStrings) { + if (line.matches(subString) == false) { + return false; + } + } + return true; + } + private void debugLogs(Path path) throws IOException { try (BufferedReader reader = Files.newBufferedReader(path)) { reader.lines().forEach(line -> logger.info(line)); @@ -102,7 +110,8 @@ protected boolean preserveClusterUponCompletion() { @Override protected final Settings restClientSettings() { - return Settings.builder().put(super.restClientSettings()) + return Settings.builder() + .put(super.restClientSettings()) // increase the timeout here to 90 seconds to handle long waits for a green // cluster health. the waits for green need to be longer than a minute to // account for delayed shards diff --git a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java index 2e35ac9581f9a..d38e1d41c6d8e 100644 --- a/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java +++ b/qa/evil-tests/src/test/java/org/elasticsearch/common/logging/EvilLoggerTests.java @@ -195,8 +195,14 @@ public void testConcurrentDeprecationLogger() throws IOException, UserException, "_deprecation.log"; final List deprecationEvents = Files.readAllLines(PathUtils.get(deprecationPath)); // we appended an integer to each log message, use that for sorting - deprecationEvents.sort(Comparator.comparingInt(s -> Integer.parseInt(s.split("message")[1]))); + Pattern pattern = Pattern.compile(".*message(\\d+)\""); + deprecationEvents.sort(Comparator.comparingInt(s -> { + Matcher matcher = pattern.matcher(s); + matcher.matches(); + return Integer.parseInt(matcher.group(1)); + })); assertThat(deprecationEvents.size(), equalTo(128)); + for (int i = 0; i < 128; i++) { assertLogLine( deprecationEvents.get(i), diff --git a/qa/logging-config/build.gradle b/qa/logging-config/build.gradle index bc4c20ba122b3..d2cda916c8883 100644 --- a/qa/logging-config/build.gradle +++ b/qa/logging-config/build.gradle @@ -32,7 +32,10 @@ testClusters.integTest { integTest.runner { nonInputProperties.systemProperty 'tests.logfile', - "${-> testClusters.integTest.singleNode().getServerLog().absolutePath.replaceAll(".json", ".log")}" + "${-> testClusters.integTest.singleNode().getServerLog().absolutePath.replaceAll("_server.json", ".log")}" + + nonInputProperties.systemProperty 'tests.jsonLogfile', + "${-> testClusters.integTest.singleNode().getServerLog()}" } test { diff --git a/qa/logging-config/custom-log4j2.properties b/qa/logging-config/custom-log4j2.properties index b225d7cd550cf..be4ffc03f20ef 100644 --- a/qa/logging-config/custom-log4j2.properties +++ b/qa/logging-config/custom-log4j2.properties @@ -1,17 +1,18 @@ - status = error -# log action execution errors for easier debugging -logger.action.name = org.elasticsearch.action -logger.action.level = debug +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n +######## Server JSON ############################ appender.rolling.type = RollingFile appender.rolling.name = rolling -appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.log -appender.rolling.layout.type = PatternLayout -appender.rolling.layout.pattern =%notEmpty{%node_name} %notEmpty{%node_and_cluster_id} %notEmpty{${sys:es.logs.cluster_name}} %m%n +appender.rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_server.json +appender.rolling.layout.type = ESJsonLayout +appender.rolling.layout.type_name = server -appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.json.gz appender.rolling.policies.type = Policies appender.rolling.policies.time.type = TimeBasedTriggeringPolicy appender.rolling.policies.time.interval = 1 @@ -26,6 +27,147 @@ appender.rolling.strategy.action.condition.type = IfFileName appender.rolling.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* appender.rolling.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize appender.rolling.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ +######## Server - old style pattern ########### +appender.rolling_old.type = RollingFile +appender.rolling_old.name = rolling_old +appender.rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.log +appender.rolling_old.layout.type = PatternLayout +appender.rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}-%d{yyyy-MM-dd}-%i.log.gz +appender.rolling_old.policies.type = Policies +appender.rolling_old.policies.time.type = TimeBasedTriggeringPolicy +appender.rolling_old.policies.time.interval = 1 +appender.rolling_old.policies.time.modulate = true +appender.rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.rolling_old.policies.size.size = 128MB +appender.rolling_old.strategy.type = DefaultRolloverStrategy +appender.rolling_old.strategy.fileIndex = nomax +appender.rolling_old.strategy.action.type = Delete +appender.rolling_old.strategy.action.basepath = ${sys:es.logs.base_path} +appender.rolling_old.strategy.action.condition.type = IfFileName +appender.rolling_old.strategy.action.condition.glob = ${sys:es.logs.cluster_name}-* +appender.rolling_old.strategy.action.condition.nested_condition.type = IfAccumulatedFileSize +appender.rolling_old.strategy.action.condition.nested_condition.exceeds = 2GB +################################################ rootLogger.level = info +rootLogger.appenderRef.console.ref = console rootLogger.appenderRef.rolling.ref = rolling +rootLogger.appenderRef.rolling_old.ref = rolling_old + +######## Deprecation JSON ####################### +appender.deprecation_rolling.type = RollingFile +appender.deprecation_rolling.name = deprecation_rolling +appender.deprecation_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.json +appender.deprecation_rolling.layout.type = ESJsonLayout +appender.deprecation_rolling.layout.type_name = deprecation +appender.deprecation_rolling.layout.esmessagefields=x-opaque-id + +appender.deprecation_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation-%i.json.gz +appender.deprecation_rolling.policies.type = Policies +appender.deprecation_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling.policies.size.size = 1GB +appender.deprecation_rolling.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling.strategy.max = 4 +################################################# +######## Deprecation - old style pattern ####### +appender.deprecation_rolling_old.type = RollingFile +appender.deprecation_rolling_old.name = deprecation_rolling_old +appender.deprecation_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecation.log +appender.deprecation_rolling_old.layout.type = PatternLayout +appender.deprecation_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.deprecation_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _deprecation-%i.log.gz +appender.deprecation_rolling_old.policies.type = Policies +appender.deprecation_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.deprecation_rolling_old.policies.size.size = 1GB +appender.deprecation_rolling_old.strategy.type = DefaultRolloverStrategy +appender.deprecation_rolling_old.strategy.max = 4 +################################################# +logger.deprecation.name = org.elasticsearch.deprecation +logger.deprecation.level = warn +logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling +logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old +logger.deprecation.additivity = false + +######## Search slowlog JSON #################### +appender.index_search_slowlog_rolling.type = RollingFile +appender.index_search_slowlog_rolling.name = index_search_slowlog_rolling +appender.index_search_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ + .cluster_name}_index_search_slowlog.json +appender.index_search_slowlog_rolling.layout.type = ESJsonLayout +appender.index_search_slowlog_rolling.layout.type_name = index_search_slowlog +appender.index_search_slowlog_rolling.layout.esmessagefields=message,took,took_millis,total_hits,types,stats,search_type,total_shards,source,id + +appender.index_search_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ + .cluster_name}_index_search_slowlog-%i.json.gz +appender.index_search_slowlog_rolling.policies.type = Policies +appender.index_search_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling.policies.size.size = 1GB +appender.index_search_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling.strategy.max = 4 +################################################# +######## Search slowlog - old style pattern #### +appender.index_search_slowlog_rolling_old.type = RollingFile +appender.index_search_slowlog_rolling_old.name = index_search_slowlog_rolling_old +appender.index_search_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_search_slowlog.log +appender.index_search_slowlog_rolling_old.layout.type = PatternLayout +appender.index_search_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.index_search_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_search_slowlog-%i.log.gz +appender.index_search_slowlog_rolling_old.policies.type = Policies +appender.index_search_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.index_search_slowlog_rolling_old.policies.size.size = 1GB +appender.index_search_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.index_search_slowlog_rolling_old.strategy.max = 4 +################################################# +logger.index_search_slowlog_rolling.name = index.search.slowlog +logger.index_search_slowlog_rolling.level = trace +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling.ref = index_search_slowlog_rolling +logger.index_search_slowlog_rolling.appenderRef.index_search_slowlog_rolling_old.ref = index_search_slowlog_rolling_old +logger.index_search_slowlog_rolling.additivity = false + +######## Indexing slowlog JSON ################## +appender.index_indexing_slowlog_rolling.type = RollingFile +appender.index_indexing_slowlog_rolling.name = index_indexing_slowlog_rolling +appender.index_indexing_slowlog_rolling.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog.json +appender.index_indexing_slowlog_rolling.layout.type = ESJsonLayout +appender.index_indexing_slowlog_rolling.layout.type_name = index_indexing_slowlog +appender.index_indexing_slowlog_rolling.layout.esmessagefields=message,took,took_millis,doc_type,id,routing,source + +appender.index_indexing_slowlog_rolling.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog-%i.json.gz +appender.index_indexing_slowlog_rolling.policies.type = Policies +appender.index_indexing_slowlog_rolling.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling.strategy.max = 4 +################################################# +######## Indexing slowlog - old style pattern ## +appender.index_indexing_slowlog_rolling_old.type = RollingFile +appender.index_indexing_slowlog_rolling_old.name = index_indexing_slowlog_rolling_old +appender.index_indexing_slowlog_rolling_old.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog.log +appender.index_indexing_slowlog_rolling_old.layout.type = PatternLayout +appender.index_indexing_slowlog_rolling_old.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +appender.index_indexing_slowlog_rolling_old.filePattern = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}\ + _index_indexing_slowlog-%i.log.gz +appender.index_indexing_slowlog_rolling_old.policies.type = Policies +appender.index_indexing_slowlog_rolling_old.policies.size.type = SizeBasedTriggeringPolicy +appender.index_indexing_slowlog_rolling_old.policies.size.size = 1GB +appender.index_indexing_slowlog_rolling_old.strategy.type = DefaultRolloverStrategy +appender.index_indexing_slowlog_rolling_old.strategy.max = 4 +################################################# + +logger.index_indexing_slowlog.name = index.indexing.slowlog.index +logger.index_indexing_slowlog.level = trace +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling.ref = index_indexing_slowlog_rolling +logger.index_indexing_slowlog.appenderRef.index_indexing_slowlog_rolling_old.ref = index_indexing_slowlog_rolling_old +logger.index_indexing_slowlog.additivity = false diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java index da75e843524da..596fd9e20cc57 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/ESJsonLayoutTests.java @@ -50,7 +50,7 @@ public void testLayout() { "\"node.name\": \"%node_name\", " + "\"message\": \"%notEmpty{%enc{%marker}{JSON} }%enc{%.-10000m}{JSON}\"" + "%notEmpty{, %node_and_cluster_id }" + - "%notEmpty{, %CustomMapFields{} }" + + "%notEmpty{, %CustomMapFields }" + "%exceptionAsJson }" + System.lineSeparator())); } @@ -71,7 +71,7 @@ public void testLayoutWithAdditionalFieldOverride() { "\"cluster.name\": \"${sys:es.logs.cluster_name}\", " + "\"node.name\": \"%node_name\"" + "%notEmpty{, %node_and_cluster_id }" + - "%notEmpty{, %CustomMapFields{message} }" + + "%notEmpty{, %CustomMapFields }" + "%exceptionAsJson }" + System.lineSeparator())); } } diff --git a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java index d7f0cadbdbe91..5ffee1b4ec417 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/common/logging/JsonLoggerTests.java @@ -38,23 +38,28 @@ import org.junit.BeforeClass; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.function.Function; +import java.util.regex.Pattern; import java.util.stream.Collectors; import java.util.stream.Stream; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.hasEntry; +import static org.hamcrest.Matchers.hasItems; import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.matchesRegex; import static org.hamcrest.Matchers.not; /** * This test confirms JSON log structure is properly formatted and can be parsed. - * It has to be in a org.elasticsearch.common.logging package to use PrefixLogger + * It has to be in a org.elasticsearch.common.logging package to use PrefixLogger. + * When running from IDE set -Dtests.security.manager=false */ public class JsonLoggerTests extends ESTestCase { @@ -62,6 +67,7 @@ public class JsonLoggerTests extends ESTestCase { @BeforeClass public static void initNodeName() { + assert "false".equals(System.getProperty("tests.security.manager")) : "-Dtests.security.manager=false has to be set"; JsonLogsTestSetup.init(); } @@ -78,6 +84,7 @@ public void tearDown() throws Exception { Configurator.shutdown(context); super.tearDown(); } + public void testDeprecatedMessage() throws IOException { final Logger testLogger = LogManager.getLogger("deprecation.test"); testLogger.info(DeprecatedMessage.of("someId","deprecated message1")); @@ -91,8 +98,8 @@ public void testDeprecatedMessage() throws IOException { assertThat(jsonLogs, contains( allOf( hasEntry("type", "deprecation"), - hasEntry("level", "INFO"), - hasEntry("component", "d.test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "deprecated message1"), @@ -102,30 +109,6 @@ public void testDeprecatedMessage() throws IOException { } } - public void testMessageOverrideWithNoValue() throws IOException { - //message field is meant to be overriden (see custom.test config), but is not provided. - //Expected is that it will be emptied - final Logger testLogger = LogManager.getLogger("custom.test"); - - testLogger.info(new ESLogMessage("some message")); - - final Path path = PathUtils.get(System.getProperty("es.logs.base_path"), - System.getProperty("es.logs.cluster_name") + "_custom.json"); - try (Stream> stream = JsonLogsStream.mapStreamFrom(path)) { - List> jsonLogs = stream - .collect(Collectors.toList()); - - assertThat(jsonLogs, contains( - allOf( - hasEntry("type", "custom"), - hasEntry("level", "INFO"), - hasEntry("component", "c.test"), - hasEntry("cluster.name", "elasticsearch"), - hasEntry("node.name", "sample-name")) - ) - ); - } - } public void testBuildingMessage() throws IOException { final Logger testLogger = LogManager.getLogger("test"); @@ -143,8 +126,8 @@ public void testBuildingMessage() throws IOException { assertThat(jsonLogs, contains( allOf( hasEntry("type", "file"), - hasEntry("level", "INFO"), - hasEntry("component", "test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "some message value0 value1"), @@ -155,39 +138,6 @@ public void testBuildingMessage() throws IOException { } } - public void testMessageOverride() throws IOException { - - final Logger testLogger = LogManager.getLogger("custom.test"); - testLogger.info(new ESLogMessage("some message") - .with("message","overriden")); - - - final Path path = PathUtils.get(System.getProperty("es.logs.base_path"), - System.getProperty("es.logs.cluster_name") + "_custom.json"); - try (Stream> stream = JsonLogsStream.mapStreamFrom(path)) { - List> jsonLogs = stream - .collect(Collectors.toList()); - - assertThat(jsonLogs, contains( - allOf( - hasEntry("type", "custom"), - hasEntry("level", "INFO"), - hasEntry("component", "c.test"), - hasEntry("cluster.name", "elasticsearch"), - hasEntry("node.name", "sample-name"), - hasEntry("message", "overriden")) - ) - ); - } - - final Path plaintextPath = PathUtils.get(System.getProperty("es.logs.base_path"), - System.getProperty("es.logs.cluster_name") + "_plaintext.json"); - List lines = Files.readAllLines(plaintextPath); - assertThat(lines, contains("some message")); - - - } - public void testCustomMessageWithMultipleFields() throws IOException { // if a field is defined to be overriden, it has to always be overriden in that appender. final Logger testLogger = LogManager.getLogger("test"); @@ -204,8 +154,8 @@ public void testCustomMessageWithMultipleFields() throws IOException { assertThat(jsonLogs, contains( allOf( hasEntry("type", "file"), - hasEntry("level", "INFO"), - hasEntry("component", "test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("field1", "value1"), @@ -233,16 +183,16 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { assertThat(jsonLogs, contains( allOf( hasEntry("type", "deprecation"), - hasEntry("level", "INFO"), - hasEntry("component", "d.test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "deprecated message1"), hasEntry("x-opaque-id", "someId")), allOf( hasEntry("type", "deprecation"), - hasEntry("level", "INFO"), - hasEntry("component", "d.test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "deprecated message2"), @@ -250,8 +200,8 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { ), allOf( hasEntry("type", "deprecation"), - hasEntry("level", "INFO"), - hasEntry("component", "d.test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "deprecated message3"), @@ -259,8 +209,8 @@ public void testDeprecatedMessageWithoutXOpaqueId() throws IOException { ), allOf( hasEntry("type", "deprecation"), - hasEntry("level", "INFO"), - hasEntry("component", "d.test"), + hasEntry("log.level", "INFO"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "deprecated message4"), @@ -304,9 +254,10 @@ public void testPrefixLoggerInJson() throws IOException { try (Stream stream = JsonLogsStream.from(path)) { List jsonLogs = collectLines(stream); assertThat(jsonLogs, contains( - logLine("file", Level.INFO, "sample-name", "p.shardIdLogger", - "[indexName][123] This is an info message with a shardId"), - logLine("file", Level.INFO, "sample-name", "p.prefixLogger", "PREFIX This is an info message with a prefix") + logLine("file", Level.INFO, "sample-name", "prefix.shardIdLogger", + "This is an info message with a shardId", Map.of(JsonLogLine::getTags, List.of("[indexName][123]"))), + logLine("file", Level.INFO, "sample-name", "prefix.prefixLogger", + "This is an info message with a prefix", Map.of(JsonLogLine::getTags, List.of("PREFIX"))) )); } } @@ -345,14 +296,14 @@ public void testStacktrace() throws IOException { assertThat(jsonLogs, contains( allOf( logLine("file", Level.ERROR, "sample-name", "test", "error message"), - stacktraceWith("java.lang.Exception: exception message"), - stacktraceWith("Caused by: java.lang.RuntimeException: cause message") + stacktraceMatches("java.lang.Exception: exception message.*Caused by: java.lang.RuntimeException: cause message.*") ) )); } } - public void testJsonInStacktraceMessageIsSplitted() throws IOException { + + public void testJsonInStacktraceMessageIsNotSplitted() throws IOException { final Logger testLogger = LogManager.getLogger("test"); String json = "{" + LINE_SEPARATOR + @@ -376,8 +327,8 @@ public void testJsonInStacktraceMessageIsSplitted() throws IOException { //message field will have a single line with json escaped logLine("file", Level.ERROR, "sample-name", "test", "error message " + json), - //stacktrace field will have each json line will in a separate array element - stacktraceWith(("java.lang.Exception: " + json).split(LINE_SEPARATOR)) + //stacktrace message will be single line + stacktraceWith("java.lang.Exception: " + json) ) )); } @@ -405,8 +356,8 @@ public void testDuplicateLogMessages() throws IOException { assertThat(jsonLogs, contains( allOf( hasEntry("type", "deprecation"), - hasEntry("level", "WARN"), - hasEntry("component", "d.test"), + hasEntry("log.level", "WARN"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "message1"), @@ -437,8 +388,8 @@ public void testDuplicateLogMessages() throws IOException { assertThat(jsonLogs, contains( allOf( hasEntry("type", "deprecation"), - hasEntry("level", "WARN"), - hasEntry("component", "d.test"), + hasEntry("log.level", "WARN"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "message1"), @@ -446,8 +397,8 @@ public void testDuplicateLogMessages() throws IOException { ), allOf( hasEntry("type", "deprecation"), - hasEntry("level", "WARN"), - hasEntry("component", "d.test"), + hasEntry("log.level", "WARN"), + hasEntry("log.logger", "deprecation.test"), hasEntry("cluster.name", "elasticsearch"), hasEntry("node.name", "sample-name"), hasEntry("message", "message1"), @@ -486,23 +437,53 @@ private void setupLogging(final String config, final Settings settings) throws I LogConfigurator.configure(environment); } + private Matcher logLine(String type, Level level, String nodeName, String component, String message) { + return logLine(mapOfParamsToCheck(type, level, nodeName, component, message)); + } + + private Map, Object> mapOfParamsToCheck( + String type, Level level, String nodeName, String component, String message) { + return Map.of(JsonLogLine::getType, type, + JsonLogLine::getLevel, level.toString(), + JsonLogLine::getNodeName, nodeName, + JsonLogLine::getComponent, component, + JsonLogLine::getMessage, message); + } + + private Matcher logLine(String type, Level level, String nodeName, String component, String message, + Map, Object> additionalProperties) { + Map, Object> map = new HashMap<>(); + map.putAll(mapOfParamsToCheck(type, level, nodeName, component, message)); + map.putAll(additionalProperties); + return logLine(map); + } + + private Matcher logLine(Map, Object> map) { return new FeatureMatcher(Matchers.is(true), "logLine", "logLine") { @Override protected Boolean featureValueOf(JsonLogLine actual) { - return Objects.equals(actual.type(), type) && - Objects.equals(actual.level(), level.toString()) && - Objects.equals(actual.nodeName(), nodeName) && - Objects.equals(actual.component(), component) && - Objects.equals(actual.message(), message); + return map.entrySet() + .stream() + .allMatch(entry -> Objects.equals(entry.getKey().apply(actual), entry.getValue())); } }; } + private Matcher stacktraceWith(String line) { + return new FeatureMatcher>(hasItems(Matchers.containsString(line)), + "error.stack_trace", "error.stack_trace") { - private Matcher stacktraceWith(String... lines) { - return new FeatureMatcher>(Matchers.hasItems(lines), - "stacktrace", "stacktrace") { + @Override + protected List featureValueOf(JsonLogLine actual) { + return actual.stacktrace(); + } + }; + } + + private Matcher stacktraceMatches(String regexp) { + return new FeatureMatcher>(hasItems(matchesRegex(Pattern.compile(regexp, Pattern.DOTALL))), + "error.stack_trace", "error.stack_trace") { @Override protected List featureValueOf(JsonLogLine actual) { @@ -510,4 +491,5 @@ protected List featureValueOf(JsonLogLine actual) { } }; } + } diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java index 6ab9a00c1e9f5..877968640855c 100644 --- a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java +++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/CustomLoggingConfigIT.java @@ -40,11 +40,20 @@ * The intention is to confirm that users can still run their Elasticsearch instances with previous configurations. */ public class CustomLoggingConfigIT extends ESRestTestCase { - private static final String NODE_STARTED = ".*integTest-0.*cluster.uuid.*node.id.*recovered.*cluster_state.*"; + //we are looking for a line where pattern contains: + // [2020-03-20T14:51:59,989][INFO ][o.e.g.GatewayService ] [integTest-0] recovered [0] indices into cluster_state + private static final String NODE_STARTED = ".*recovered.*cluster_state.*"; public void testSuccessfulStartupWithCustomConfig() throws Exception { assertBusy(() -> { - List lines = readAllLines(getLogFile()); + List lines = readAllLines(getPlaintextLogFile()); + assertThat(lines, Matchers.hasItem(RegexMatcher.matches(NODE_STARTED))); + }); + } + + public void testParseAllV7JsonLines() throws Exception { + assertBusy(() -> { + List lines = readAllLines(getJSONLogFile()); assertThat(lines, Matchers.hasItem(RegexMatcher.matches(NODE_STARTED))); }); } @@ -60,7 +69,18 @@ private List readAllLines(Path logFile) { } @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") - private Path getLogFile() { + private Path getJSONLogFile() { + String logFileString = System.getProperty("tests.logfile"); + if (logFileString == null) { + fail("tests.logfile must be set to run this test. It is automatically " + + "set by gradle. If you must set it yourself then it should be the absolute path to the " + + "log file."); + } + return Paths.get(logFileString); + } + + @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") + private Path getPlaintextLogFile() { String logFileString = System.getProperty("tests.logfile"); if (logFileString == null) { fail("tests.logfile must be set to run this test. It is automatically " diff --git a/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/ESJsonLogsConfigIT.java b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/ESJsonLogsConfigIT.java new file mode 100644 index 0000000000000..07d6d8127cf29 --- /dev/null +++ b/qa/logging-config/src/test/java/org/elasticsearch/qa/custom_logging/ESJsonLogsConfigIT.java @@ -0,0 +1,67 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.qa.custom_logging; + +import org.elasticsearch.common.logging.JsonLogLine; +import org.elasticsearch.common.logging.JsonLogsIntegTestCase; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.hamcrest.Matcher; + +import java.io.BufferedReader; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.AccessController; +import java.security.PrivilegedAction; + +import static org.hamcrest.Matchers.is; + +/** + * Test to verify ES JSON log format. Used in ES v7. Some users might decide to keep that format. + */ +public class ESJsonLogsConfigIT extends JsonLogsIntegTestCase { + @Override + protected Matcher nodeNameMatcher() { + return is("integTest-0"); + } + + @Override + protected BufferedReader openReader(Path logFile) { + assumeFalse("Skipping test because it is being run against an external cluster.", + logFile.getFileName().toString().equals("--external--")); + return AccessController.doPrivileged((PrivilegedAction) () -> { + try { + return Files.newBufferedReader(logFile, StandardCharsets.UTF_8); + } catch (IOException e) { + throw new RuntimeException(e); + } + }); + } + + @Override + protected String getLogFileName() { + return System.getProperty("tests.jsonLogfile"); + } + + @Override + protected ObjectParser getParser() { + return JsonLogLine.ES_LOG_LINE; + } +} diff --git a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties index 5c2a158403457..46c91d4d89b8e 100644 --- a/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties +++ b/qa/logging-config/src/test/resources/org/elasticsearch/common/logging/json_layout/log4j2.properties @@ -1,40 +1,27 @@ appender.console.type = Console appender.console.name = console -appender.console.layout.type = ESJsonLayout +appender.console.layout.type = ECSJsonLayout appender.console.layout.type_name = console + appender.file.type = File appender.file.name = file appender.file.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}.json -appender.file.layout.type = ESJsonLayout +appender.file.layout.type = ECSJsonLayout appender.file.layout.type_name = file appender.deprecated.type = File appender.deprecated.name = deprecated appender.deprecated.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs.cluster_name}_deprecated.json -appender.deprecated.layout.type = ESJsonLayout +appender.deprecated.layout.type = ECSJsonLayout appender.deprecated.layout.type_name = deprecation appender.deprecatedconsole.type = Console appender.deprecatedconsole.name = deprecatedconsole -appender.deprecatedconsole.layout.type = ESJsonLayout +appender.deprecatedconsole.layout.type = ECSJsonLayout appender.deprecatedconsole.layout.type_name = deprecation -appender.customconsole.type = Console -appender.customconsole.name = customconsole -appender.customconsole.layout.type = ESJsonLayout -appender.customconsole.layout.type_name = custom -appender.customconsole.layout.overrideFields = message - -appender.custom.type = File -appender.custom.name = custom -appender.custom.fileName = ${sys:es.logs.base_path}${sys:file.separator}${sys:es.logs\ - .cluster_name}_custom.json -appender.custom.layout.type = ESJsonLayout -appender.custom.layout.type_name = custom -appender.custom.layout.overrideFields = message - rootLogger.level = info rootLogger.appenderRef.console.ref = console @@ -52,13 +39,6 @@ logger.deprecation.appenderRef.deprecatedconsole.ref = deprecatedconsole logger.deprecation.additivity = false -logger.custom.name = custom.test -logger.custom.level = trace -logger.custom.appenderRef.console.ref = custom -logger.custom.appenderRef.file.ref = customconsole -logger.custom.appenderRef.plaintext.ref = plaintext -logger.custom.additivity = false - logger.test.name = test logger.test.level = trace logger.test.appenderRef.console.ref = console diff --git a/qa/logging-config/src/test/resources/plugin-security.policy b/qa/logging-config/src/test/resources/plugin-security.policy index d0d865c4ede16..1dd8051b7ff37 100644 --- a/qa/logging-config/src/test/resources/plugin-security.policy +++ b/qa/logging-config/src/test/resources/plugin-security.policy @@ -1,4 +1,5 @@ grant { // Needed to read the log file permission java.io.FilePermission "${tests.logfile}", "read"; + permission java.io.FilePermission "${tests.jsonLogfile}", "read"; }; diff --git a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java index d45ae29e3d15e..a67ef7ba4249a 100644 --- a/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java +++ b/qa/os/src/test/java/org/elasticsearch/packaging/test/DockerTests.java @@ -570,7 +570,7 @@ public void test120DockerLogsIncludeElasticsearchLogs() throws Exception { waitForElasticsearch(installation); final Result containerLogs = getContainerLogs(); - assertThat("Container logs don't contain abbreviated class names", containerLogs.stdout, containsString("o.e.n.Node")); + assertThat("Container logs should contain full class names", containerLogs.stdout, containsString("org.elasticsearch.node.Node")); assertThat("Container logs don't contain INFO level messages", containerLogs.stdout, containsString("INFO")); } diff --git a/server/build.gradle b/server/build.gradle index 59c41c7679d20..0954ab45fa431 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -123,6 +123,10 @@ dependencies { // repackaged jna with native bits linked against all elastic supported platforms compile "org.elasticsearch:jna:${versions.jna}" + + compile "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}" + compile "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}" + if (!isEclipse) { java12Compile sourceSets.main.output } @@ -212,6 +216,7 @@ thirdPartyAudit.ignoreMissingClasses( // from log4j 'com.conversantmedia.util.concurrent.DisruptorBlockingQueue', 'com.conversantmedia.util.concurrent.SpinPolicy', + 'com.fasterxml.jackson.databind.SerializationFeature', 'com.fasterxml.jackson.annotation.JsonInclude$Include', 'com.fasterxml.jackson.databind.DeserializationContext', 'com.fasterxml.jackson.databind.DeserializationFeature', diff --git a/server/licenses/ecs-logging-core-0.1.3.jar.sha1 b/server/licenses/ecs-logging-core-0.1.3.jar.sha1 new file mode 100644 index 0000000000000..c21c3dabd7d0c --- /dev/null +++ b/server/licenses/ecs-logging-core-0.1.3.jar.sha1 @@ -0,0 +1 @@ +e0f93944011bf9e9ba147a8b7e14aad2584b9123 \ No newline at end of file diff --git a/server/licenses/ecs-logging-core-LICENSE.txt b/server/licenses/ecs-logging-core-LICENSE.txt new file mode 100644 index 0000000000000..ad37dfadf0db5 --- /dev/null +++ b/server/licenses/ecs-logging-core-LICENSE.txt @@ -0,0 +1,1829 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ecs-logging-java/LICENSE at b5ba2398eec9f6febf6239da638ecf79f4a54b5d · elastic/ecs-logging-java · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + + +
+ +
+ + +
+ +
+ + + +
+
+
+ + + + + + + + + + + + + + + + + +
+
+ + + +

+ + /ecs-logging-java + + +

+ +
+ + + +
+ +
+ + +
+
+
+ + + + + + + + + Permalink + + + + + +
+ + +
+ + Tree: + b5ba2398ee + + + + + + + +
+ +
+ + Find file + + + Copy path + +
+ +

+ ecs-logging-java/LICENSE +

+ +
+ + Find file + + + Copy path + +
+
+ + + +
+
+
+ +

+ elastic/ecs-logging-java is licensed under the +

+

Apache License 2.0

+

A permissive license whose main conditions require preservation of copyright and license notices. Contributors provide an express grant of patent rights. Licensed works, modifications, and larger works may be distributed under different terms and without source code.

+
+ +
+
+

Permissions

+
    +
  • + + + Commercial use + +
    • +
  • + + + Modification + +
    • +
  • + + + Distribution + +
    • +
  • + + + Patent use + +
    • +
  • + + + Private use + +
    • +
+
+
+

Limitations

+
    +
  • + + + Trademark use + +
    • +
  • + + + Liability + +
    • +
  • + + + Warranty + +
    • +
+
+
+

Conditions

+
    +
  • + + + License and copyright notice + +
    • +
  • + + + State changes + +
    • +
+
+
+
+

+ This is not legal advice. + Learn more about repository licenses. +

+
+ + + + +
+
+ + @felixbarny + felixbarny + + License headers and other legal stuff (#6) + + + + a859c6b + Aug 16, 2019 + +
+ +
+
+ + 1 contributor + + +
+ +

+ Users who have contributed to this file +

+
+ + +
+
+
+ + + + + +
+ +
+
+ + 201 lines (169 sloc) + + 11.1 KB +
+ +
+ +
+ Raw + Blame + History +
+ + +
+ + + +
+
+
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
1. Definitions.
+
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
+
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
+
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
+
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
+
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
+
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
+
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
+
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
+
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
+
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
+
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
+
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
+
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
+
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
+
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
+
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
+
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
+
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
+
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
+
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
+
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
+
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
+
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
+
END OF TERMS AND CONDITIONS
+
APPENDIX: How to apply the Apache License to your work.
+
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
+
Copyright 2019 Elastic and contributors
+
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
+
http://www.apache.org/licenses/LICENSE-2.0
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+ + + +
+ +
+ + + +
+ + +
+ + +
 +
+ + + +
+
+ +
+
+ + +
+ + + + + + +
+ + + You can’t perform that action at this time. +
+ + + + + + + + + + + + + + +
+ + + + diff --git a/server/licenses/ecs-logging-core-NOTICE.txt b/server/licenses/ecs-logging-core-NOTICE.txt new file mode 100644 index 0000000000000..cbe7f44e3184c --- /dev/null +++ b/server/licenses/ecs-logging-core-NOTICE.txt @@ -0,0 +1,1081 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ecs-logging-java/NOTICE at b5ba2398eec9f6febf6239da638ecf79f4a54b5d · elastic/ecs-logging-java · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + + +
+ +
+ + +
+ +
+ + + +
+
+
+ + + + + + + + + + + + + + + + + +
+
+ + + +

+ + /ecs-logging-java + + +

+ +
+ + + +
+ +
+ + +
+
+
+ + + + + + + + + Permalink + + + + + +
+ + +
+ + Tree: + b5ba2398ee + + + + + + + +
+ +
+ + Find file + + + Copy path + +
+ +

+ ecs-logging-java/NOTICE +

+ +
+ + Find file + + + Copy path + +
+
+ + + + + + +
+
+ + @felixbarny + felixbarny + + License headers and other legal stuff (#6) + + + + a859c6b + Aug 16, 2019 + +
+ +
+
+ + 1 contributor + + +
+ +

+ Users who have contributed to this file +

+
+ + +
+
+
+ + + + + +
+ +
+
+ + 43 lines (32 sloc) + + 1.77 KB +
+ +
+ +
+ Raw + Blame + History +
+ + +
+ + + +
+
+
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
java-ecs-logging
Copyright 2019 Elasticsearch B.V.
+
###############################################################################
This product includes software licensed under the Apache License 2.0 developed at FasterXML.
+
Jackson LICENSE:
-------------------------------------------------------------------------------
This copy of Jackson JSON processor streaming parser/generator is licensed under the
Apache (Software) License, version 2.0 ("the License").
See the License for details about distribution rights, and the
specific rights regarding derivate works.
+
You may obtain a copy of the License at:
+
http://www.apache.org/licenses/LICENSE-2.0
-------------------------------------------------------------------------------
+
+
Jackson NOTICE:
-------------------------------------------------------------------------------
# Jackson JSON processor
+
Jackson is a high-performance, Free/Open Source JSON processing library.
It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
been in development since 2007.
It is currently developed by a community of developers, as well as supported
commercially by FasterXML.com.
+
## Licensing
+
Jackson core and extension components may licensed under different licenses.
To find the details that apply to this artifact see the accompanying LICENSE file.
For more information, including possible other licensing options, contact
FasterXML.com (http://fasterxml.com).
+
## Credits
+
A list of contributors may be found from CREDITS file, which is included
in some artifacts (usually source distributions); but is always available
from the source code management (SCM) system project uses.
-------------------------------------------------------------------------------
###############################################################################
+ + + +
+ +
+ + + +
+ + +
+ + +
 +
+ + + +
+
+ +
+
+ + +
+ + + + + + +
+ + + You can’t perform that action at this time. +
+ + + + + + + + + + + + + + +
+ + + + diff --git a/server/licenses/log4j2-ecs-layout-0.1.3.jar.sha1 b/server/licenses/log4j2-ecs-layout-0.1.3.jar.sha1 new file mode 100644 index 0000000000000..02e3c5af3698d --- /dev/null +++ b/server/licenses/log4j2-ecs-layout-0.1.3.jar.sha1 @@ -0,0 +1 @@ +5a46cc50b82d8bfbd2156a82558edc0eecb75821 \ No newline at end of file diff --git a/server/licenses/log4j2-ecs-layout-LICENSE.txt b/server/licenses/log4j2-ecs-layout-LICENSE.txt new file mode 100644 index 0000000000000..ad37dfadf0db5 --- /dev/null +++ b/server/licenses/log4j2-ecs-layout-LICENSE.txt @@ -0,0 +1,1829 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ecs-logging-java/LICENSE at b5ba2398eec9f6febf6239da638ecf79f4a54b5d · elastic/ecs-logging-java · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + + +
+ +
+ + +
+ +
+ + + +
+
+
+ + + + + + + + + + + + + + + + + +
+
+ + + +

+ + /ecs-logging-java + + +

+ +
+ + + +
+ +
+ + +
+
+
+ + + + + + + + + Permalink + + + + + +
+ + +
+ + Tree: + b5ba2398ee + + + + + + + +
+ +
+ + Find file + + + Copy path + +
+ +

+ ecs-logging-java/LICENSE +

+ +
+ + Find file + + + Copy path + +
+
+ + + +
+
+
+ +

+ elastic/ecs-logging-java is licensed under the +

+

Apache License 2.0

+

A permissive license whose main conditions require preservation of copyright and license notices. Contributors provide an express grant of patent rights. Licensed works, modifications, and larger works may be distributed under different terms and without source code.

+
+ +
+
+

Permissions

+
    +
  • + + + Commercial use + +
    • +
  • + + + Modification + +
    • +
  • + + + Distribution + +
    • +
  • + + + Patent use + +
    • +
  • + + + Private use + +
    • +
+
+
+

Limitations

+
    +
  • + + + Trademark use + +
    • +
  • + + + Liability + +
    • +
  • + + + Warranty + +
    • +
+
+
+

Conditions

+
    +
  • + + + License and copyright notice + +
    • +
  • + + + State changes + +
    • +
+
+
+
+

+ This is not legal advice. + Learn more about repository licenses. +

+
+ + + + +
+
+ + @felixbarny + felixbarny + + License headers and other legal stuff (#6) + + + + a859c6b + Aug 16, 2019 + +
+ +
+
+ + 1 contributor + + +
+ +

+ Users who have contributed to this file +

+
+ + +
+
+
+ + + + + +
+ +
+
+ + 201 lines (169 sloc) + + 11.1 KB +
+ +
+ +
+ Raw + Blame + History +
+ + +
+ + + +
+
+
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
1. Definitions.
+
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
+
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
+
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
+
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
+
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
+
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
+
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
+
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
+
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
+
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
+
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
+
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
+
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
+
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
+
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
+
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
+
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
+
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
+
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
+
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
+
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
+
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
+
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
+
END OF TERMS AND CONDITIONS
+
APPENDIX: How to apply the Apache License to your work.
+
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
+
Copyright 2019 Elastic and contributors
+
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
+
http://www.apache.org/licenses/LICENSE-2.0
+
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+ + + +
+ +
+ + + +
+ + +
+ + +
 +
+ + + +
+
+ +
+
+ + +
+ + + + + + +
+ + + You can’t perform that action at this time. +
+ + + + + + + + + + + + + + +
+ + + + diff --git a/server/licenses/log4j2-ecs-layout-NOTICE.txt b/server/licenses/log4j2-ecs-layout-NOTICE.txt new file mode 100644 index 0000000000000..cbe7f44e3184c --- /dev/null +++ b/server/licenses/log4j2-ecs-layout-NOTICE.txt @@ -0,0 +1,1081 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ecs-logging-java/NOTICE at b5ba2398eec9f6febf6239da638ecf79f4a54b5d · elastic/ecs-logging-java · GitHub + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + + + + + +
+ +
+ + +
+ +
+ + + +
+
+
+ + + + + + + + + + + + + + + + + +
+
+ + + +

+ + /ecs-logging-java + + +

+ +
+ + + +
+ +
+ + +
+
+
+ + + + + + + + + Permalink + + + + + +
+ + +
+ + Tree: + b5ba2398ee + + + + + + + +
+ +
+ + Find file + + + Copy path + +
+ +

+ ecs-logging-java/NOTICE +

+ +
+ + Find file + + + Copy path + +
+
+ + + + + + +
+
+ + @felixbarny + felixbarny + + License headers and other legal stuff (#6) + + + + a859c6b + Aug 16, 2019 + +
+ +
+
+ + 1 contributor + + +
+ +

+ Users who have contributed to this file +

+
+ + +
+
+
+ + + + + +
+ +
+
+ + 43 lines (32 sloc) + + 1.77 KB +
+ +
+ +
+ Raw + Blame + History +
+ + +
+ + + +
+
+
+ + + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
java-ecs-logging
Copyright 2019 Elasticsearch B.V.
+
###############################################################################
This product includes software licensed under the Apache License 2.0 developed at FasterXML.
+
Jackson LICENSE:
-------------------------------------------------------------------------------
This copy of Jackson JSON processor streaming parser/generator is licensed under the
Apache (Software) License, version 2.0 ("the License").
See the License for details about distribution rights, and the
specific rights regarding derivate works.
+
You may obtain a copy of the License at:
+
http://www.apache.org/licenses/LICENSE-2.0
-------------------------------------------------------------------------------
+
+
Jackson NOTICE:
-------------------------------------------------------------------------------
# Jackson JSON processor
+
Jackson is a high-performance, Free/Open Source JSON processing library.
It was originally written by Tatu Saloranta (tatu.saloranta@iki.fi), and has
been in development since 2007.
It is currently developed by a community of developers, as well as supported
commercially by FasterXML.com.
+
## Licensing
+
Jackson core and extension components may licensed under different licenses.
To find the details that apply to this artifact see the accompanying LICENSE file.
For more information, including possible other licensing options, contact
FasterXML.com (http://fasterxml.com).
+
## Credits
+
A list of contributors may be found from CREDITS file, which is included
in some artifacts (usually source distributions); but is always available
from the source code management (SCM) system project uses.
-------------------------------------------------------------------------------
###############################################################################
+ + + +
+ +
+ + + +
+ + +
+ + +
 +
+ + + +
+
+ +
+
+ + +
+ + + + + + +
+ + + You can’t perform that action at this time. +
+ + + + + + + + + + + + + + +
+ + + + diff --git a/server/src/main/java/org/elasticsearch/common/logging/ClusterIdConverter.java b/server/src/main/java/org/elasticsearch/common/logging/ClusterIdConverter.java new file mode 100644 index 0000000000000..90f24c8c7917f --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/ClusterIdConverter.java @@ -0,0 +1,59 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.pattern.ConverterKeys; +import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; +import org.apache.logging.log4j.core.pattern.PatternConverter; + +/** + * Pattern converter to format the cluster_id variable into JSON fields cluster.id. + */ +@Plugin(category = PatternConverter.CATEGORY, name = "ClusterIdConverter") +@ConverterKeys({"cluster_id"}) +public final class ClusterIdConverter extends LogEventPatternConverter { + /** + * Called by log4j2 to initialize this converter. + */ + public static ClusterIdConverter newInstance(@SuppressWarnings("unused") final String[] options) { + return new ClusterIdConverter(); + } + + public ClusterIdConverter() { + super("cluster_id", "cluster_id"); + } + + /** + * Formats the cluster.uuid into json fields. + * + * @param event - a log event is ignored in this method as it uses the clusterId value + * from NodeAndClusterIdStateListener to format + */ + @Override + public void format(LogEvent event, StringBuilder toAppendTo) { + if (NodeAndClusterIdStateListener.nodeAndClusterId.get() != null) { + toAppendTo.append(NodeAndClusterIdStateListener.nodeAndClusterId.get().v2()); + } + // nodeId/clusterUuid not received yet, not appending + } + +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/CustomMapFieldsConverter.java b/server/src/main/java/org/elasticsearch/common/logging/CustomMapFieldsConverter.java index 0ac5bf029c389..8aa4e5bcd2809 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/CustomMapFieldsConverter.java +++ b/server/src/main/java/org/elasticsearch/common/logging/CustomMapFieldsConverter.java @@ -26,8 +26,6 @@ import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; import org.apache.logging.log4j.core.pattern.PatternConverter; -import java.util.Set; - /** * Pattern converter to populate CustomMapFields in a pattern. * This is to be used with custom ElasticSearch log messages @@ -37,32 +35,22 @@ @ConverterKeys({"CustomMapFields"}) public final class CustomMapFieldsConverter extends LogEventPatternConverter { - - private Set overridenFields; - - public CustomMapFieldsConverter(Set overridenFields) { + public CustomMapFieldsConverter() { super("CustomMapFields", "CustomMapFields"); - this.overridenFields = overridenFields; } /** * Called by log4j2 to initialize this converter. */ public static CustomMapFieldsConverter newInstance(final Configuration config, final String[] options) { - Set overridenFields = csvToSet(options[0]); - return new CustomMapFieldsConverter(overridenFields); - } - - private static Set csvToSet(String csv) { - String[] split = csv.split(","); - return Set.of(split); + return new CustomMapFieldsConverter(); } @Override public void format(LogEvent event, StringBuilder toAppendTo) { if(event.getMessage() instanceof ESLogMessage) { ESLogMessage logMessage = (ESLogMessage) event.getMessage(); - logMessage.asJson(toAppendTo); + logMessage.addJsonNoBrackets(toAppendTo); } } } diff --git a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java index 7a03cd934266c..e0ea1f74f92e4 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java +++ b/server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java @@ -19,6 +19,7 @@ package org.elasticsearch.common.logging; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.elasticsearch.common.Strings; import org.elasticsearch.common.SuppressLoggerChecks; @@ -35,7 +36,15 @@ public static ESLogMessage of(String xOpaqueId, String messagePattern, Object... return new ESLogMessage(messagePattern, args); } - return new ESLogMessage(messagePattern, args) - .with(X_OPAQUE_ID_FIELD_NAME, xOpaqueId); + Object value = new Object() { + @Override + public String toString() { + return ParameterizedMessage.format(messagePattern, args); + + } + }; + return new ESLogMessage() + .field("message", value) + .field(X_OPAQUE_ID_FIELD_NAME, xOpaqueId); } } diff --git a/server/src/main/java/org/elasticsearch/common/logging/ECSJsonLayout.java b/server/src/main/java/org/elasticsearch/common/logging/ECSJsonLayout.java new file mode 100644 index 0000000000000..b7d4ddd3f1231 --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/ECSJsonLayout.java @@ -0,0 +1,85 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import co.elastic.logging.log4j2.EcsLayout; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.config.Node; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginBuilderFactory; +import org.apache.logging.log4j.core.layout.AbstractStringLayout; +import org.apache.logging.log4j.core.util.KeyValuePair; + +import java.nio.charset.StandardCharsets; + +/** + * This is a wrapper class around co.elastic.logging.log4j2.EcsLayout + * in order to avoid a duplication of configuration in log4j2.properties + */ +@Plugin(name = "ECSJsonLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) +public class ECSJsonLayout { + + @PluginBuilderFactory + public static ECSJsonLayout.Builder newBuilder() { + return new ECSJsonLayout.Builder().asBuilder(); + } + + public static class Builder extends AbstractStringLayout.Builder + implements org.apache.logging.log4j.core.util.Builder { + + @PluginAttribute("type_name") + String type; + + public Builder() { + setCharset(StandardCharsets.UTF_8); + } + + @Override + public EcsLayout build() { + return EcsLayout.newBuilder() + .setConfiguration(getConfiguration()) + .setServiceName("ES_ECS") + .setStackTraceAsArray(false) + .setIncludeMarkers(true) + .setAdditionalFields(additionalFields()) + .build(); + } + + private KeyValuePair[] additionalFields() { + return new KeyValuePair[]{ + new KeyValuePair("type",type), + new KeyValuePair("cluster.uuid","%cluster_id"), + new KeyValuePair("node.id","%node_id"), + new KeyValuePair("node.name","%ESnode_name"), + new KeyValuePair("cluster.name","${sys:es.logs.cluster_name}"), + }; + } + + public String getType() { + return type; + } + + public Builder setType(final String type) { + this.type = type; + return asBuilder(); + } + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java b/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java index 0bccc15e1d9da..83959a3bc762b 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java +++ b/server/src/main/java/org/elasticsearch/common/logging/ESJsonLayout.java @@ -50,7 +50,7 @@ *
  • component - logger name, most of the times class name
    • *
  • cluster.name - taken from sys:es.logs.cluster_name system property because it is always set
    • *
  • node.name - taken from NodeNamePatternConverter, as it can be set in runtime as hostname when not set in elasticsearch.yml
    • - *
  • node_and_cluster_id - in json as node.id and cluster.uuid - taken from NodeAndClusterIdConverter and present + *
  • node_and_cluster_id - in json as node.id and cluster.uuid - taken from NodeIdConverter and present * once clusterStateUpdate is first received
    • *
  • message - a json escaped message. Multiline messages will be converted to single line with new line explicitly * replaced to \n
    • @@ -66,23 +66,25 @@ * Once an appender is defined to be overriding a field, all the log events should contain this field. *

    * The value taken from ESLogMessage has to be a simple escaped JSON value. + * @deprecated ECSJsonlayout should be used as JSON logs layout */ +@Deprecated(since = "v8") @Plugin(name = "ESJsonLayout", category = Node.CATEGORY, elementType = Layout.ELEMENT_TYPE, printObject = true) public class ESJsonLayout extends AbstractStringLayout { private final PatternLayout patternLayout; - private String overridenFields; + private String esmessagefields; protected ESJsonLayout(String typeName, Charset charset, String[] overrideFields) { super(charset); - this.overridenFields = String.join(",",overrideFields); + this.esmessagefields = String.join(",",overrideFields); this.patternLayout = PatternLayout.newBuilder() .withPattern(pattern(typeName, overrideFields)) .withAlwaysWriteExceptions(false) .build(); } - private String pattern(String type, String[] overrideFields) { + private String pattern(String type, String[] esmessagefields) { if (Strings.isEmpty(type)) { throw new IllegalArgumentException("layout parameter 'type_name' cannot be empty"); } @@ -96,27 +98,23 @@ private String pattern(String type, String[] overrideFields) { map.put("message", inQuotes("%notEmpty{%enc{%marker}{JSON} }%enc{%.-10000m}{JSON}")); - // overridden fields are expected to be present in a log message - for (String key : overrideFields) { + // esmessagefields are treated as potentially overriding + for (String key : esmessagefields) { map.remove(key); } - return createPattern(map, Set.of(overrideFields)); + return createPattern(map, Set.of(esmessagefields)); } - private String createPattern(Map map, Set overrideFields) { + private String createPattern(Map map, Set esmessagefields) { StringBuilder sb = new StringBuilder(); sb.append("{"); String separator = ""; for (Map.Entry entry : map.entrySet()) { - if (overrideFields.contains(entry.getKey())) { - sb.append("%notEmpty{"); - sb.append(separator); - appendField(sb, entry); - sb.append("}"); - } else { + // fields present in esmessagefields are meant to be provided in CustomMapFields + if (esmessagefields.contains(entry.getKey()) == false) { sb.append(separator); appendField(sb, entry); } @@ -124,7 +122,7 @@ private String createPattern(Map map, Set overrideFields separator = ", "; } sb.append(notEmpty(", %node_and_cluster_id ")); - sb.append(notEmpty(", %CustomMapFields{"+overridenFields+"} ")); + sb.append(notEmpty(", %CustomMapFields ")); sb.append("%exceptionAsJson "); sb.append("}"); sb.append(System.lineSeparator()); @@ -169,7 +167,7 @@ public static class Builder> extends AbstractS @PluginAttribute(value = "charset", defaultString = "UTF-8") Charset charset; - @PluginAttribute("overrideFields") + @PluginAttribute("esmessagefields") private String overrideFields; public Builder() { diff --git a/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java b/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java index 6aaf5f1bd862b..47e05fe44f8c7 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java +++ b/server/src/main/java/org/elasticsearch/common/logging/ESLogMessage.java @@ -35,23 +35,32 @@ * A base class for custom log4j logger messages. Carries additional fields which will populate JSON fields in logs. */ public class ESLogMessage extends MapMessage { - private final String messagePattern; private final List arguments = new ArrayList<>(); - public ESLogMessage(String messagePattern, Object... arguments) { + public ESLogMessage(String messagePattern, Object... args) { + super(new LinkedHashMap<>()); + Collections.addAll(this.arguments, args); + Object message = new Object() { + @Override + public String toString() { + return ParameterizedMessage.format(messagePattern, arguments.toArray()); + } + }; + with("message", message); + } + + public ESLogMessage() { super(new LinkedHashMap<>()); - this.messagePattern = messagePattern; - Collections.addAll(this.arguments, arguments); } public ESLogMessage argAndField(String key, Object value) { this.arguments.add(value); - super.with(key,value); + super.with(key, value); return this; } public ESLogMessage field(String key, Object value) { - super.with(key,value); + super.with(key, value); return this; } @@ -60,15 +69,12 @@ public ESLogMessage withFields(Map prepareMap) { return this; } - @Override - protected void appendMap(final StringBuilder sb) { - String message = ParameterizedMessage.format(messagePattern, arguments.toArray()); - sb.append(message); - } - - //taken from super.asJson without the wrapping '{' '}' - @Override - protected void asJson(StringBuilder sb) { + /** + * This method is used in order to support ESJsonLayout which replaces %CustomMapFields from a pattern with JSON fields + * It is a modified version of {@link MapMessage#asJson(StringBuilder)} where the curly brackets are not added + * @param sb a string builder where JSON fields will be attached + */ + protected void addJsonNoBrackets(StringBuilder sb) { for (int i = 0; i < getIndexedReadOnlyStringMap().size(); i++) { if (i > 0) { sb.append(", "); diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java index 27437947870b4..181664057c0c5 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java +++ b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdConverter.java @@ -24,39 +24,29 @@ import org.apache.logging.log4j.core.pattern.ConverterKeys; import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; import org.apache.logging.log4j.core.pattern.PatternConverter; -import org.apache.lucene.util.SetOnce; import java.util.Locale; /** * Pattern converter to format the node_and_cluster_id variable into JSON fields node.id and cluster.uuid. * Keeping those two fields together assures that they will be atomically set and become visible in logs at the same time. + * + * @deprecated this class is kept in order to allow working log configuration from 7.x */ @Plugin(category = PatternConverter.CATEGORY, name = "NodeAndClusterIdConverter") @ConverterKeys({"node_and_cluster_id"}) +@Deprecated public final class NodeAndClusterIdConverter extends LogEventPatternConverter { - private static final SetOnce nodeAndClusterId = new SetOnce<>(); - - /** - * Called by log4j2 to initialize this converter. - */ - public static NodeAndClusterIdConverter newInstance(@SuppressWarnings("unused") final String[] options) { - return new NodeAndClusterIdConverter(); - } public NodeAndClusterIdConverter() { super("NodeAndClusterId", "node_and_cluster_id"); } /** - * Updates only once the clusterID and nodeId. - * Subsequent executions will throw {@link org.apache.lucene.util.SetOnce.AlreadySetException}. - * - * @param nodeId a nodeId received from cluster state update - * @param clusterUUID a clusterId received from cluster state update + * Called by log4j2 to initialize this converter. */ - public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) { - nodeAndClusterId.set(formatIds(clusterUUID, nodeId)); + public static NodeAndClusterIdConverter newInstance(@SuppressWarnings("unused") final String[] options) { + return new NodeAndClusterIdConverter(); } /** @@ -66,13 +56,15 @@ public static void setNodeIdAndClusterId(String nodeId, String clusterUUID) { */ @Override public void format(LogEvent event, StringBuilder toAppendTo) { - if (nodeAndClusterId.get() != null) { - toAppendTo.append(nodeAndClusterId.get()); + if (NodeAndClusterIdStateListener.nodeAndClusterId.get() != null) { + String nodeId = NodeAndClusterIdStateListener.nodeAndClusterId.get().v1(); + String clusterUUID = NodeAndClusterIdStateListener.nodeAndClusterId.get().v2(); + toAppendTo.append(formatIds(nodeId, clusterUUID)); } // nodeId/clusterUuid not received yet, not appending } - private static String formatIds(String clusterUUID, String nodeId) { + private String formatIds(String nodeId, String clusterUUID) { return String.format(Locale.ROOT, "\"cluster.uuid\": \"%s\", \"node.id\": \"%s\"", clusterUUID, nodeId); } } diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java index e8f636238447a..84052ec944332 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java +++ b/server/src/main/java/org/elasticsearch/common/logging/NodeAndClusterIdStateListener.java @@ -21,25 +21,28 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.lucene.util.SetOnce; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.ClusterStateObserver; import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.util.concurrent.ThreadContext; /** * The {@link NodeAndClusterIdStateListener} listens to cluster state changes and ONLY when receives the first update - * it sets the clusterUUID and nodeID in log4j pattern converter {@link NodeAndClusterIdConverter}. + * it sets the clusterUUID and nodeID in log4j pattern converter {@link NodeIdConverter}. * Once the first update is received, it will automatically be de-registered from subsequent updates. */ public class NodeAndClusterIdStateListener implements ClusterStateObserver.Listener { private static final Logger logger = LogManager.getLogger(NodeAndClusterIdStateListener.class); + static final SetOnce> nodeAndClusterId = new SetOnce<>(); private NodeAndClusterIdStateListener() {} /** * Subscribes for the first cluster state update where nodeId and clusterId is present - * and sets these values in {@link NodeAndClusterIdConverter}. + * and sets these values in {@link NodeIdConverter}. */ public static void getAndSetNodeIdAndClusterId(ClusterService clusterService, ThreadContext threadContext) { ClusterState clusterState = clusterService.state(); @@ -66,7 +69,11 @@ public void onNewClusterState(ClusterState state) { String clusterUUID = getClusterUUID(state); logger.debug("Received cluster state update. Setting nodeId=[{}] and clusterUuid=[{}]", nodeId, clusterUUID); - NodeAndClusterIdConverter.setNodeIdAndClusterId(nodeId, clusterUUID); + setNodeIdAndClusterId(nodeId, clusterUUID); + } + + void setNodeIdAndClusterId(String nodeId, String clusterUUID){ + nodeAndClusterId.set(Tuple.tuple(nodeId,clusterUUID)); } @Override diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeIdConverter.java b/server/src/main/java/org/elasticsearch/common/logging/NodeIdConverter.java new file mode 100644 index 0000000000000..9ff44dadb54ae --- /dev/null +++ b/server/src/main/java/org/elasticsearch/common/logging/NodeIdConverter.java @@ -0,0 +1,58 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.common.logging; + +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.pattern.ConverterKeys; +import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; +import org.apache.logging.log4j.core.pattern.PatternConverter; + +/** + * Pattern converter to format the node_id variable into JSON fields node.id . + */ +@Plugin(category = PatternConverter.CATEGORY, name = "NodeIdConverter") +@ConverterKeys({"node_id"}) +public final class NodeIdConverter extends LogEventPatternConverter { + /** + * Called by log4j2 to initialize this converter. + */ + public static NodeIdConverter newInstance(@SuppressWarnings("unused") final String[] options) { + return new NodeIdConverter(); + } + + public NodeIdConverter() { + super("node_id", "node_id"); + } + + /** + * Formats the node.id into json fields. + * + * @param event - a log event is ignored in this method as it uses the clusterId value + * from NodeAndClusterIdStateListener to format + */ + @Override + public void format(LogEvent event, StringBuilder toAppendTo) { + if (NodeAndClusterIdStateListener.nodeAndClusterId.get() != null) { + toAppendTo.append(NodeAndClusterIdStateListener.nodeAndClusterId.get().v1()); + } + // nodeId/clusterUuid not received yet, not appending + } +} diff --git a/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java b/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java index 43b34f532a23b..98f7ef4b008d7 100644 --- a/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java +++ b/server/src/main/java/org/elasticsearch/common/logging/NodeNamePatternConverter.java @@ -19,15 +19,16 @@ package org.elasticsearch.common.logging; -import java.util.Arrays; - import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.config.plugins.Plugin; import org.apache.logging.log4j.core.pattern.ConverterKeys; import org.apache.logging.log4j.core.pattern.LogEventPatternConverter; import org.apache.logging.log4j.core.pattern.PatternConverter; import org.apache.lucene.util.SetOnce; +import java.util.Arrays; + /** * Converts {@code %node_name} in log4j patterns into the current node name. * We can't use a system property for this because the node name system @@ -35,7 +36,7 @@ * elasticsearch.yml. */ @Plugin(category = PatternConverter.CATEGORY, name = "NodeNamePatternConverter") -@ConverterKeys({"node_name"}) +@ConverterKeys({"ESnode_name","node_name"}) public final class NodeNamePatternConverter extends LogEventPatternConverter { /** * The name of this node. @@ -49,6 +50,11 @@ static void setNodeName(String nodeName) { NODE_NAME.set(nodeName); } + public static void setGlobalNodeName(String nodeName){ + LoggerContext ctx = LoggerContext.getContext(false); + ctx.getConfiguration().getProperties().put("node_name",nodeName); + } + /** * Called by log4j2 to initialize this converter. */ diff --git a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java index 61b5c8f1acdb4..c965f68fa937e 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java +++ b/server/src/main/java/org/elasticsearch/index/IndexingSlowLog.java @@ -167,10 +167,7 @@ public static ESLogMessage of( Index index, ParsedDocument doc, long tookInNanos, boolean reformat, int maxSourceCharsToLog) { Map jsonFields = prepareMap(index, doc, tookInNanos, reformat, maxSourceCharsToLog); - //message for json logs is provided by jsonFields - String plaintextMessage = message(index, doc, tookInNanos, reformat, maxSourceCharsToLog); - - return new ESLogMessage(plaintextMessage).withFields(jsonFields); + return new ESLogMessage().withFields(jsonFields); } private static Map prepareMap(Index index, ParsedDocument doc, long tookInNanos, boolean reformat, @@ -206,36 +203,6 @@ private static Map prepareMap(Index index, ParsedDocument doc, l } return map; } - - private static String message(Index index, ParsedDocument doc, long tookInNanos, boolean reformat, int maxSourceCharsToLog) { - StringBuilder sb = new StringBuilder(); - sb.append(index).append(" "); - sb.append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], "); - sb.append("took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], "); - sb.append("id[").append(doc.id()).append("], "); - if (doc.routing() == null) { - sb.append("routing[]"); - } else { - sb.append("routing[").append(doc.routing()).append("]"); - } - - if (maxSourceCharsToLog == 0 || doc.source() == null || doc.source().length() == 0) { - return sb.toString(); - } - try { - String source = XContentHelper.convertToJson(doc.source(), reformat, doc.getXContentType()); - sb.append(", source[").append(Strings.cleanTruncate(source, maxSourceCharsToLog).trim()).append("]"); - } catch (IOException e) { - sb.append(", source[_failed_to_convert_[").append(e.getMessage()).append("]]"); - /* - * We choose to fail to write to the slow log and instead let this percolate up to the post index listener loop where this - * will be logged at the warn level. - */ - final String message = String.format(Locale.ROOT, "failed to convert source for slow log entry [%s]", sb.toString()); - throw new UncheckedIOException(message, e); - } - return sb.toString(); - } } boolean isReformat() { diff --git a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java index d93c707eadb05..98616edd8d95c 100644 --- a/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java +++ b/server/src/main/java/org/elasticsearch/index/SearchSlowLog.java @@ -22,7 +22,6 @@ import com.fasterxml.jackson.core.io.JsonStringEncoder; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.elasticsearch.common.Strings; import org.elasticsearch.common.logging.ESLogMessage; import org.elasticsearch.common.logging.Loggers; import org.elasticsearch.common.settings.Setting; @@ -157,10 +156,7 @@ static final class SearchSlowLogMessage { public static ESLogMessage of(SearchContext context, long tookInNanos) { Map jsonFields = prepareMap(context, tookInNanos); - // message for json logs is overridden from json Fields - String plaintextMessage = message(context, tookInNanos); - return new ESLogMessage(plaintextMessage) - .withFields(jsonFields); + return new ESLogMessage().withFields(jsonFields); } private static Map prepareMap(SearchContext context, long tookInNanos) { @@ -190,42 +186,6 @@ private static Map prepareMap(SearchContext context, long tookIn return messageFields; } - // Message will be used in plaintext logs - private static String message(SearchContext context, long tookInNanos) { - StringBuilder sb = new StringBuilder(); - sb.append(context.indexShard().shardId()) - .append(" ") - .append("took[").append(TimeValue.timeValueNanos(tookInNanos)).append("], ") - .append("took_millis[").append(TimeUnit.NANOSECONDS.toMillis(tookInNanos)).append("], ") - .append("total_hits["); - if (context.queryResult().getTotalHits() != null) { - sb.append(context.queryResult().getTotalHits()); - } else { - sb.append("-1"); - } - sb.append("], "); - if (context.groupStats() == null) { - sb.append("stats[], "); - } else { - sb.append("stats["); - Strings.collectionToDelimitedString(context.groupStats(), ",", "", "", sb); - sb.append("], "); - } - sb.append("search_type[").append(context.searchType()).append("], total_shards[") - .append(context.numberOfShards()).append("], "); - if (context.request().source() != null) { - sb.append("source[").append(context.request().source().toString(FORMAT_PARAMS)).append("], "); - } else { - sb.append("source[], "); - } - if (context.getTask().getHeader(Task.X_OPAQUE_ID) != null) { - sb.append("id[").append(context.getTask().getHeader(Task.X_OPAQUE_ID)).append("], "); - } else { - sb.append("id[], "); - } - return sb.toString(); - } - private static String escapeJson(String text) { byte[] sourceEscaped = JsonStringEncoder.getInstance().quoteAsUTF8(text); return new String(sourceEscaped, UTF_8); diff --git a/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java b/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java index 111e94eb2b050..b9119f62cf960 100644 --- a/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java +++ b/server/src/test/java/org/elasticsearch/common/logging/JsonThrowablePatternConverterTests.java @@ -70,7 +70,7 @@ public void testStacktraceWithJson() throws IOException { //confirms exception is correctly parsed - JsonLogLine jsonLogLine = JsonLogsStream.from(new BufferedReader(new StringReader(result))) + JsonLogLine jsonLogLine = JsonLogsStream.from(new BufferedReader(new StringReader(result)), JsonLogLine.ES_LOG_LINE) .findFirst() .orElseThrow(() -> new AssertionError("no logs parsed")); diff --git a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java index 7047b4799f12c..b5e9ded7dcd46 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexingSlowLogTests.java @@ -83,8 +83,6 @@ public void testEmptyRoutingField() throws IOException { ESLogMessage p = IndexingSlowLogMessage.of(index, pd, 10, true, 0); assertThat(p.get("routing"), nullValue()); - - assertThat(p.asString(), containsString("routing[]")); } public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { @@ -99,16 +97,17 @@ public void testSlowLogParsedDocumentPrinterSourceToLog() throws IOException { // Turning on document logging logs the whole thing p = IndexingSlowLogMessage.of(index, pd, 10, true, Integer.MAX_VALUE); - assertThat(p.getFormattedMessage(), containsString("source[{\"foo\":\"bar\"}]")); + assertThat(p.get("source"), equalTo("{\\\"foo\\\":\\\"bar\\\"}")); // And you can truncate the source p = IndexingSlowLogMessage.of(index, pd, 10, true, 3); - assertThat(p.getFormattedMessage(), containsString("source[{\"f]")); + assertThat(p.get("source"), equalTo("{\\\"f")); // And you can truncate the source p = IndexingSlowLogMessage.of(index, pd, 10, true, 3); - assertThat(p.getFormattedMessage(), containsString("source[{\"f]")); - assertThat(p.getFormattedMessage(), startsWith("[foo/123] took")); + assertThat(p.get("source"), containsString("{\\\"f")); + assertThat(p.get("message"), startsWith("[foo/123]")); + assertThat(p.get("took"), containsString("10nanos")); // Throwing a error if source cannot be converted source = new BytesArray("invalid"); diff --git a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java index 63424cf2b2aca..259f1bee1e2cd 100644 --- a/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java +++ b/server/src/test/java/org/elasticsearch/index/SearchSlowLogTests.java @@ -43,12 +43,10 @@ import java.util.List; import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasToString; import static org.hamcrest.Matchers.instanceOf; import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.startsWith; public class SearchSlowLogTests extends ESSingleNodeTestCase { @Override @@ -120,10 +118,10 @@ public void testSlowLogSearchContextPrinterToLog() throws IOException { searchContext.setTask(new SearchShardTask(0, "n/a", "n/a", "test", null, Collections.singletonMap(Task.X_OPAQUE_ID, "my_id"))); ESLogMessage p = SearchSlowLog.SearchSlowLogMessage.of(searchContext, 10); - assertThat(p.getFormattedMessage(), startsWith("[foo][0]")); + assertThat(p.get("message"), equalTo("[foo][0]")); // Makes sure that output doesn't contain any new lines - assertThat(p.getFormattedMessage(), not(containsString("\n"))); - assertThat(p.getFormattedMessage(), endsWith("id[my_id], ")); + assertThat(p.get("source"), not(containsString("\n"))); + assertThat(p.get("id"), equalTo("my_id")); } public void testLevelSetting() { diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java index 15f4b17ba6f8c..820f3147bb611 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java +++ b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogLine.java @@ -30,7 +30,9 @@ * Parsing log lines with this class confirms the json format of logs */ public class JsonLogLine { - public static final ObjectParser PARSER = createParser(true); + public static final ObjectParser ECS_LOG_LINE = createECSParser(true); + public static final ObjectParser ES_LOG_LINE = createESParser(true); + private String type; private String timestamp; @@ -41,6 +43,7 @@ public class JsonLogLine { private String clusterUuid; private String nodeId; private String message; + private List tags; private List stacktrace; @Override @@ -55,47 +58,52 @@ public String toString() { sb.append(", clusterUuid='").append(clusterUuid).append('\''); sb.append(", nodeId='").append(nodeId).append('\''); sb.append(", message='").append(message).append('\''); + sb.append(", tags='").append(tags).append('\''); sb.append(", stacktrace=").append(stacktrace); sb.append('}'); return sb.toString(); } - public String type() { + public String getType() { return type; } - public String timestamp() { + public String getTimestamp() { return timestamp; } - public String level() { + public String getLevel() { return level; } - public String component() { + public String getComponent() { return component; } - public String clusterName() { + public String getClusterName() { return clusterName; } - public String nodeName() { + public String getNodeName() { return nodeName; } - public String clusterUuid() { + public String getClusterUuid() { return clusterUuid; } - public String nodeId() { + public String getNodeId() { return nodeId; } - public String message() { + public String getMessage() { return message; } + public List getTags() { + return tags; + } + public List stacktrace() { return stacktrace; } @@ -136,11 +144,32 @@ public void setMessage(String message) { this.message = message; } + public void setTags(List tags) { + this.tags = tags; + } + public void setStacktrace(List stacktrace) { this.stacktrace = stacktrace; } - private static ObjectParser createParser(boolean ignoreUnknownFields) { + private static ObjectParser createECSParser(boolean ignoreUnknownFields) { + ObjectParser parser = new ObjectParser<>("json_log_line", ignoreUnknownFields, JsonLogLine::new); + parser.declareString(JsonLogLine::setType, new ParseField("type")); + parser.declareString(JsonLogLine::setTimestamp, new ParseField("@timestamp")); + parser.declareString(JsonLogLine::setLevel, new ParseField("log.level")); + parser.declareString(JsonLogLine::setComponent, new ParseField("log.logger")); + parser.declareString(JsonLogLine::setClusterName, new ParseField("cluster.name")); + parser.declareString(JsonLogLine::setNodeName, new ParseField("node.name")); + parser.declareString(JsonLogLine::setClusterUuid, new ParseField("cluster.uuid")); + parser.declareString(JsonLogLine::setNodeId, new ParseField("node.id")); + parser.declareString(JsonLogLine::setMessage, new ParseField("message")); + parser.declareStringArray(JsonLogLine::setTags, new ParseField("tags")); + parser.declareStringArray(JsonLogLine::setStacktrace, new ParseField("error.stack_trace")); + + return parser; + } + + private static ObjectParser createESParser(boolean ignoreUnknownFields) { ObjectParser parser = new ObjectParser<>("search_template", ignoreUnknownFields, JsonLogLine::new); parser.declareString(JsonLogLine::setType, new ParseField("type")); parser.declareString(JsonLogLine::setTimestamp, new ParseField("timestamp")); diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java index d9ba80d6b35de..6b3adc85ed532 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsIntegTestCase.java @@ -20,6 +20,7 @@ package org.elasticsearch.common.logging; import org.elasticsearch.common.SuppressForbidden; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.test.rest.ESRestTestCase; import java.io.BufferedReader; @@ -69,18 +70,18 @@ public void testElementsPresentOnAllLinesOfLog() throws IOException { JsonLogLine firstLine = findFirstLine(); assertNotNull(firstLine); - try (Stream stream = JsonLogsStream.from(openReader(getLogFile()))) { + try (Stream stream = JsonLogsStream.from(openReader(getLogFile()), getParser() )) { stream.limit(LINES_TO_CHECK) .forEach(jsonLogLine -> { - assertThat(jsonLogLine.type(), not(isEmptyOrNullString())); - assertThat(jsonLogLine.timestamp(), not(isEmptyOrNullString())); - assertThat(jsonLogLine.level(), not(isEmptyOrNullString())); - assertThat(jsonLogLine.component(), not(isEmptyOrNullString())); - assertThat(jsonLogLine.message(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.getType(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.getTimestamp(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.getLevel(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.getComponent(), not(isEmptyOrNullString())); + assertThat(jsonLogLine.getMessage(), not(isEmptyOrNullString())); // all lines should have the same nodeName and clusterName - assertThat(jsonLogLine.nodeName(), nodeNameMatcher()); - assertThat(jsonLogLine.clusterName(), equalTo(firstLine.clusterName())); + assertThat(jsonLogLine.getNodeName(), nodeNameMatcher()); + assertThat(jsonLogLine.getClusterName(), equalTo(firstLine.getClusterName())); }); } } @@ -99,7 +100,7 @@ public void testNodeIdAndClusterIdConsistentOnceAvailable() throws IOException { JsonLogLine firstLine = null; while (iterator.hasNext()) { JsonLogLine jsonLogLine = iterator.next(); - if (jsonLogLine.nodeId() != null) { + if (jsonLogLine.getNodeId() != null) { firstLine = jsonLogLine; } } @@ -110,15 +111,15 @@ public void testNodeIdAndClusterIdConsistentOnceAvailable() throws IOException { int i = 0; while (iterator.hasNext() && i++ < LINES_TO_CHECK) { JsonLogLine jsonLogLine = iterator.next(); - assertThat(jsonLogLine.nodeId(), equalTo(firstLine.nodeId())); - assertThat(jsonLogLine.clusterUuid(), equalTo(firstLine.clusterUuid())); + assertThat(jsonLogLine.getNodeId(), equalTo(firstLine.getNodeId())); + assertThat(jsonLogLine.getClusterUuid(), equalTo(firstLine.getClusterUuid())); } } } @SuppressForbidden(reason = "PathUtils doesn't have permission to read this file") private Path getLogFile() { - String logFileString = System.getProperty("tests.logfile"); + String logFileString = getLogFileName(); if (logFileString == null) { fail("tests.logfile must be set to run this test. It is automatically " + "set by gradle. If you must set it yourself then it should be the absolute path to the " @@ -126,4 +127,12 @@ private Path getLogFile() { } return Paths.get(logFileString); } + + protected String getLogFileName() { + return System.getProperty("tests.logfile"); + } + + protected ObjectParser getParser() { + return JsonLogLine.ECS_LOG_LINE; + } } diff --git a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java index ad6fedca2d928..1b92cd68785a0 100644 --- a/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java +++ b/test/framework/src/main/java/org/elasticsearch/common/logging/JsonLogsStream.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.xcontent.DeprecationHandler; import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; @@ -44,15 +45,21 @@ public class JsonLogsStream { private final XContentParser parser; private final BufferedReader reader; + private final ObjectParser logLineParser; - private JsonLogsStream(BufferedReader reader) throws IOException { + private JsonLogsStream(BufferedReader reader, ObjectParser logLineParser) throws IOException { this.reader = reader; this.parser = JsonXContent.jsonXContent.createParser(NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, reader); + this.logLineParser = logLineParser; + } + + public static Stream from(BufferedReader reader, ObjectParser logLineParser) throws IOException { + return new JsonLogsStream(reader, logLineParser).stream(); } public static Stream from(BufferedReader reader) throws IOException { - return new JsonLogsStream(reader).stream(); + return new JsonLogsStream(reader, JsonLogLine.ECS_LOG_LINE).stream(); } public static Stream from(Path path) throws IOException { @@ -60,7 +67,7 @@ public static Stream from(Path path) throws IOException { } public static Stream> mapStreamFrom(Path path) throws IOException { - return new JsonLogsStream(Files.newBufferedReader(path)).streamMap(); + return new JsonLogsStream(Files.newBufferedReader(path), JsonLogLine.ECS_LOG_LINE).streamMap(); } private Stream stream() { @@ -113,7 +120,7 @@ public boolean hasNext() { @Override public JsonLogLine next() { - JsonLogLine apply = JsonLogLine.PARSER.apply(parser, null); + JsonLogLine apply = logLineParser.apply(parser, null); nextToken(); return apply; } diff --git a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java index 9767a92c2663d..52651242ea355 100644 --- a/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java +++ b/x-pack/plugin/ccr/qa/downgrade-to-basic-license/src/test/java/org/elasticsearch/xpack/ccr/FollowIndexIT.java @@ -101,11 +101,12 @@ private Matcher autoFollowCoordinatorWarn() { @Override protected Boolean featureValueOf(JsonLogLine actual) { - return actual.level().equals("WARN") && - actual.component().equals("o.e.x.c.a.AutoFollowCoordinator") && - actual.nodeName().startsWith("follow-cluster-0") && - actual.message().contains("failure occurred while fetching cluster state for auto follow pattern [test_pattern]") && - actual.stacktrace().contains("org.elasticsearch.ElasticsearchStatusException: can not fetch remote cluster state " + + return actual.getLevel().equals("WARN") && + actual.getComponent().contains("AutoFollowCoordinator") && + actual.getNodeName().startsWith("follow-cluster-0") && + actual.getMessage().contains("failure occurred while fetching cluster state for auto follow pattern [test_pattern]") && + actual.stacktrace().get(0) + .contains("org.elasticsearch.ElasticsearchStatusException: can not fetch remote cluster state " + "as the remote cluster [leader_cluster] is not licensed for [ccr]; the license mode [BASIC]" + " on cluster [leader_cluster] does not enable [ccr]"); }