Force ECS and JSON logging for libbeat/logp

CHANGELOG-developer.next.asciidoc

@@ -56,6 +56,7 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove Metricbeat EventFetcher and EventsFetcher interface. Use the reporter interface instead. {pull}25093[25093]
 - Update Darwin build image to a debian 10 base that increases the MacOS SDK and minimum supported version used in build to 10.14. {issue}24193[24193]
 - Removed the `common.Float` type. {issue}28279[28279] {pull}28280[28280] {pull}28376[28376]
+- libbeat.logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 
 ==== Bugfixes
 

CHANGELOG.next.asciidoc

@@ -26,6 +26,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Improve stats API {pull}27963[27963]
 - Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
 - Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
+- Libbeat: logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
 - Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https://github.com/golang/go/issues/46123 {pull}27543[27543]
 
 *Auditbeat*

auditbeat/auditbeat.reference.yml

@@ -1448,14 +1448,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

auditbeat/tests/system/test_file_integrity.py

@@ -98,7 +98,7 @@ def test_non_recursive(self):
 
             # wait until file1 is reported before deleting. Otherwise the hash
             # might not be calculated
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file1)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file1)), ignore_case=True)
 
             os.unlink(file1)
 
@@ -107,8 +107,9 @@ def test_non_recursive(self):
             file3 = os.path.join(subdir, "other_file.txt")
             self.create_file(file3, "not reported.")
 
-            self.wait_log_contains("\"deleted\"")
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(subdir)), ignore_case=True)
+            # log entries are JSON formatted, this value shows up as an escaped json string.
+            self.wait_log_contains("\\\"deleted\\\"")
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(subdir)), ignore_case=True)
             self.wait_output(3)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir.lower() for obj in self.read_output()))
@@ -157,7 +158,7 @@ def test_recursive(self):
             # wait until the directories to watch are printed in the logs
             # this happens when the file_integrity module starts
             self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True)
-            self.wait_log_contains("\"recursive\": true")
+            self.wait_log_contains("\"recursive\":true")
 
             # auditbeat_test/subdir/
             subdir = os.path.join(dirs[0], "subdir")
@@ -173,7 +174,7 @@ def test_recursive(self):
             file2 = os.path.join(subdir2, "more.txt")
             self.create_file(file2, "")
 
-            self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file2)), ignore_case=True)
+            self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file2)), ignore_case=True)
             self.wait_output(4)
             self.wait_until(lambda: any(
                 'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output()))

filebeat/filebeat.reference.yml

@@ -2360,14 +2360,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Filebeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

heartbeat/heartbeat.reference.yml

@@ -1594,14 +1594,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Heartbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

journalbeat/journalbeat.reference.yml

@@ -1391,14 +1391,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Journalbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

libbeat/_meta/config/logging.reference.yml.tmpl

@@ -72,11 +72,3 @@ logging.files:
   # renamed during rotation. Or when set to date, the date is added to
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
-
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false

libbeat/docs/loggingconfig.asciidoc

@@ -264,17 +264,6 @@ If the log file already exists on startup, immediately rotate it and start
 writing to a new file instead of appending to the existing one. Defaults to
 true.
 
-[float]
-==== `logging.json`
-
-When true, logs messages in JSON format. The default is false.
-
-[float]
-==== `logging.ecs`
-
-When true, logs messages with minimal required Elastic Common Schema (ECS)
-information.
-
 ifndef::serverless[]
 [float]
 ==== `logging.files.redirect_stderr` experimental[]

libbeat/logp/config.go

@@ -26,11 +26,9 @@ import (
 // Config contains the configuration options for the logger. To create a Config
 // from a common.Config use logp/config.Build.
 type Config struct {
-	Beat       string   `config:",ignore"`        // Name of the Beat (for default file name).
-	JSON       bool     `config:"json"`           // Write logs as JSON.
-	Level      Level    `config:"level"`          // Logging level (error, warning, info, debug).
-	Selectors  []string `config:"selectors"`      // Selectors for debug level logging.
-	ECSEnabled bool     `config:"ecs" yaml:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line
+	Beat      string   `config:",ignore"`   // Name of the Beat (for default file name).
+	Level     Level    `config:"level"`     // Logging level (error, warning, info, debug).
+	Selectors []string `config:"selectors"` // Selectors for debug level logging.
 
 	toObserver  bool
 	toIODiscard bool

libbeat/logp/core.go

@@ -196,7 +196,7 @@ func makeOptions(cfg Config) []zap.Option {
 	if cfg.development {
 		options = append(options, zap.Development())
 	}
-	if cfg.ECSEnabled {
+	if cfg.Beat != "" {
 		fields := []zap.Field{
 			zap.String("service.name", cfg.Beat),
 		}
@@ -254,10 +254,7 @@ func newCore(cfg Config, enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapco
 	return wrappedCore(cfg, zapcore.NewCore(enc, ws, enab))
 }
 func wrappedCore(cfg Config, core zapcore.Core) zapcore.Core {
-	if cfg.ECSEnabled {
-		return ecszap.WrapCore(core)
-	}
-	return core
+	return ecszap.WrapCore(core)
 }
 
 func globalLogger() *zap.Logger {

libbeat/logp/core_test.go

@@ -48,7 +48,7 @@ func TestLogger(t *testing.T) {
 
 	TestingSetup()
 	exerciseLogger()
-	TestingSetup(AsJSON())
+	TestingSetup()
 	exerciseLogger()
 }
 
@@ -152,7 +152,6 @@ func TestLoggingECSFields(t *testing.T) {
 		Beat:        "beat1",
 		Level:       DebugLevel,
 		development: true,
-		ECSEnabled:  true,
 		Files: FileConfig{
 			Name: "beat1.log",
 		},

libbeat/logp/encoding.go

@@ -43,20 +43,15 @@ type encoderCreator func(cfg zapcore.EncoderConfig) zapcore.Encoder
 func buildEncoder(cfg Config) zapcore.Encoder {
 	var encCfg zapcore.EncoderConfig
 	var encCreator encoderCreator
-	if cfg.JSON {
-		encCfg = JSONEncoderConfig()
-		encCreator = zapcore.NewJSONEncoder
-	} else if cfg.ToSyslog {
+	if cfg.ToSyslog {
 		encCfg = SyslogEncoderConfig()
 		encCreator = zapcore.NewConsoleEncoder
 	} else {
-		encCfg = ConsoleEncoderConfig()
-		encCreator = zapcore.NewConsoleEncoder
+		encCfg = JSONEncoderConfig()
+		encCreator = zapcore.NewJSONEncoder
 	}
 
-	if cfg.ECSEnabled {
-		encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
-	}
+	encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
 	return encCreator(encCfg)
 }
 

libbeat/logp/options.go

@@ -52,10 +52,3 @@ func ToDiscardOutput() Option {
 		cfg.ToStderr = false
 	}
 }
-
-// AsJSON specifies to log the output as JSON.
-func AsJSON() Option {
-	return func(cfg *Config) {
-		cfg.JSON = true
-	}
-}

libbeat/tests/system/test_logging.py

@@ -25,43 +25,25 @@ def assert_contains_ecs_log(self, logfile=None):
         assert self.log_contains(ecs_message_log, logfile=logfile)
         assert self.log_contains(ecs_log_level_log, logfile=logfile)
 
-    def assert_not_contains_ecs_log(self, logfile=None):
-        assert not self.log_contains(ecs_version_log, logfile=logfile)
-        assert not self.log_contains(ecs_timestamp_log, logfile=logfile)
-        assert not self.log_contains(ecs_log_level_log, logfile=logfile)
-
-    def test_console_default(self):
-        """
-        logs to console with default format
-        """
-        self.run_beat_with_args("mockbeat start running", logging_args=["-e"])
-        self.assert_not_contains_ecs_log()
-
     def test_console_ecs(self):
         """
         logs to console with ECS format
         """
         self.run_beat_with_args("mockbeat start running",
-                                logging_args=["-e"],
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=true"])
+                                logging_args=["-e"])
         self.assert_contains_ecs_log()
 
     def test_file_default(self):
         """
         logs to file with default format
         """
         self.run_beat_with_args("Mockbeat is alive!",
-                                logging_args=[],
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=false"])
-        self.assert_not_contains_ecs_log(logfile="logs/mockbeat")
+                                logging_args=[])
+        self.assert_contains_ecs_log(logfile="logs/mockbeat")
 
     def test_file_ecs(self):
         """
         logs to file with ECS format
         """
-        self.run_beat_with_args("Mockbeat is alive!",
-                                extra_args=["-E", "logging.json=true",
-                                            "-E", "logging.ecs=true"])
+        self.run_beat_with_args("Mockbeat is alive!")
         self.assert_contains_ecs_log(logfile="logs/mockbeat")

libbeat/tests/system/test_template.py

@@ -107,7 +107,7 @@ def test_json_template(self):
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
         self.wait_until(lambda: self.log_contains("Loading json template from file"))
-        self.wait_until(lambda: self.log_contains('Template with name "bla" loaded.'))
+        self.wait_until(lambda: self.log_contains('Template with name \\\"bla\\\" loaded.'))
         proc.check_kill_and_wait()
 
         result = es.transport.perform_request('GET', '/_template/' + template_name)
@@ -149,7 +149,7 @@ def test_template_default(self):
         self.render_config()
         proc = self.start_beat()
         self.wait_until(lambda: self.log_contains("mockbeat start running."))
-        self.wait_until(lambda: self.log_contains('Template with name "mockbeat-9.9.9" loaded.'))
+        self.wait_until(lambda: self.log_contains('Template with name \\\"mockbeat-9.9.9\\\" loaded.'))
         self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
         proc.check_kill_and_wait()
 

metricbeat/metricbeat.reference.yml

@@ -2263,14 +2263,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Metricbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

metricbeat/module/elasticsearch/test_elasticsearch.py

@@ -135,7 +135,7 @@ def test_xpack_cluster_stats(self):
             }
         }])
         proc = self.start_beat()
-        self.wait_log_contains('"dataset": "elasticsearch.cluster.stats"')
+        self.wait_log_contains('\\"dataset\\": \\"elasticsearch.cluster.stats\\"')
 
         proc.check_kill_and_wait()
         self.assert_no_logged_warnings()

packetbeat/packetbeat.reference.yml

@@ -1943,14 +1943,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Packetbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

winlogbeat/winlogbeat.reference.yml

@@ -1371,14 +1371,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Winlogbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/auditbeat/auditbeat.reference.yml

@@ -1504,14 +1504,6 @@ logging.files:
   # the end of the file. On rotation a new file is created, older files are untouched.
   #suffix: count
 
-# Set to true to log messages in JSON format.
-#logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#logging.ecs: false
-
 # ============================= X-Pack Monitoring ==============================
 # Auditbeat can export internal metrics to a central Elasticsearch monitoring
 # cluster.  This requires xpack monitoring to be enabled in Elasticsearch.  The

x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl

@@ -197,9 +197,4 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 {{template "providers.yml.tmpl" .}}

x-pack/elastic-agent/_meta/config/common.reference.p2.yml.tmpl

@@ -197,9 +197,4 @@ agent.logging.to_stderr: true
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
 
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false
-
 {{template "providers.yml.tmpl" .}}

x-pack/elastic-agent/_meta/elastic-agent.yml

@@ -191,8 +191,3 @@ agent.logging.to_stderr: true
 
 # Set to true to log messages in JSON format.
 #agent.logging.json: false
-
-# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
-# information. Recommended to use in combination with `logging.json=true`
-# Defaults to false.
-#agent.logging.ecs: false