Skip to content

Commit

Permalink
Force ECS and JSON logging for libbeat/logp (#28573) (#28918)
Browse files Browse the repository at this point in the history
Force the logp package to act as if ECS and JSON has been enabled. Attach the
service.name field to entries only when Config.Beat is not empty.
Remove the logp.Config.ECSEnabled field. Log entries will now be
JSON formatted by default.

(cherry picked from commit 987250d)

Co-authored-by: Michel Laterman <[email protected]>
  • Loading branch information
mergify[bot] and michel-laterman authored Nov 10, 2021
1 parent a4b233f commit b110f73
Show file tree
Hide file tree
Showing 33 changed files with 24 additions and 224 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Improve stats API {pull}27963[27963]
- Enable IMDSv2 support for `add_cloud_metadata` processor on AWS. {issue}22101[22101] {pull}28285[28285]
- Update kubernetes.namespace from keyword to group field and add name, labels, annotations, uuid as its fields {pull}27917[27917]
- Libbeat: logp package forces ECS compliant logs. Logs are JSON formatted. Options to enable ECS/JSON have been removed. {issue}15544[15544] {pull}28573[28573]
- Previously, RE2 and thus Golang had a bug where `(|a)*` matched more characters than `(|a)+`. To stay consistent with PCRE, the bug was fixed. Configurations that rely on the old, buggy behaviour has to be adjusted. See more about Golang bug: https://github.com/golang/go/issues/46123 {pull}27543[27543]
- Update docker client. {pull}28716[28716]
- Remove `auto` from the available options of `setup.ilm.enabled` and set the default value to `true`. {pull}28671[28671]
Expand Down
8 changes: 0 additions & 8 deletions auditbeat/auditbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1448,14 +1448,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Auditbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
11 changes: 6 additions & 5 deletions auditbeat/tests/system/test_file_integrity.py
Original file line number Diff line number Diff line change
Expand Up @@ -98,7 +98,7 @@ def test_non_recursive(self):

# wait until file1 is reported before deleting. Otherwise the hash
# might not be calculated
self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file1)), ignore_case=True)
self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file1)), ignore_case=True)

os.unlink(file1)

Expand All @@ -107,8 +107,9 @@ def test_non_recursive(self):
file3 = os.path.join(subdir, "other_file.txt")
self.create_file(file3, "not reported.")

self.wait_log_contains("\"deleted\"")
self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(subdir)), ignore_case=True)
# log entries are JSON formatted, this value shows up as an escaped json string.
self.wait_log_contains("\\\"deleted\\\"")
self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(subdir)), ignore_case=True)
self.wait_output(3)
self.wait_until(lambda: any(
'file.path' in obj and obj['file.path'].lower() == subdir.lower() for obj in self.read_output()))
Expand Down Expand Up @@ -157,7 +158,7 @@ def test_recursive(self):
# wait until the directories to watch are printed in the logs
# this happens when the file_integrity module starts
self.wait_log_contains(escape_path(dirs[0]), max_timeout=30, ignore_case=True)
self.wait_log_contains("\"recursive\": true")
self.wait_log_contains("\"recursive\":true")

# auditbeat_test/subdir/
subdir = os.path.join(dirs[0], "subdir")
Expand All @@ -173,7 +174,7 @@ def test_recursive(self):
file2 = os.path.join(subdir2, "more.txt")
self.create_file(file2, "")

self.wait_log_contains("\"path\": \"{0}\"".format(escape_path(file2)), ignore_case=True)
self.wait_log_contains("\"path\":\"{0}\"".format(escape_path(file2)), ignore_case=True)
self.wait_output(4)
self.wait_until(lambda: any(
'file.path' in obj and obj['file.path'].lower() == subdir2.lower() for obj in self.read_output()))
Expand Down
8 changes: 0 additions & 8 deletions filebeat/filebeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2360,14 +2360,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Filebeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
8 changes: 0 additions & 8 deletions heartbeat/heartbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1594,14 +1594,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Heartbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
8 changes: 0 additions & 8 deletions journalbeat/journalbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1391,14 +1391,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Journalbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
8 changes: 0 additions & 8 deletions libbeat/_meta/config/logging.reference.yml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,3 @@ logging.files:
# renamed during rotation. Or when set to date, the date is added to
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false
11 changes: 0 additions & 11 deletions libbeat/docs/loggingconfig.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -264,17 +264,6 @@ If the log file already exists on startup, immediately rotate it and start
writing to a new file instead of appending to the existing one. Defaults to
true.

[float]
==== `logging.json`

When true, logs messages in JSON format. The default is false.

[float]
==== `logging.ecs`

When true, logs messages with minimal required Elastic Common Schema (ECS)
information.

ifndef::serverless[]
[float]
==== `logging.files.redirect_stderr` experimental[]
Expand Down
8 changes: 3 additions & 5 deletions libbeat/logp/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,9 @@ import (
// Config contains the configuration options for the logger. To create a Config
// from a common.Config use logp/config.Build.
type Config struct {
Beat string `config:",ignore"` // Name of the Beat (for default file name).
JSON bool `config:"json"` // Write logs as JSON.
Level Level `config:"level"` // Logging level (error, warning, info, debug).
Selectors []string `config:"selectors"` // Selectors for debug level logging.
ECSEnabled bool `config:"ecs" yaml:"ecs"` // Adds minimal ECS information using ECS conformant keys to every log line
Beat string `config:",ignore"` // Name of the Beat (for default file name).
Level Level `config:"level"` // Logging level (error, warning, info, debug).
Selectors []string `config:"selectors"` // Selectors for debug level logging.

toObserver bool
toIODiscard bool
Expand Down
7 changes: 2 additions & 5 deletions libbeat/logp/core.go
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ func makeOptions(cfg Config) []zap.Option {
if cfg.development {
options = append(options, zap.Development())
}
if cfg.ECSEnabled {
if cfg.Beat != "" {
fields := []zap.Field{
zap.String("service.name", cfg.Beat),
}
Expand Down Expand Up @@ -254,10 +254,7 @@ func newCore(cfg Config, enc zapcore.Encoder, ws zapcore.WriteSyncer, enab zapco
return wrappedCore(cfg, zapcore.NewCore(enc, ws, enab))
}
func wrappedCore(cfg Config, core zapcore.Core) zapcore.Core {
if cfg.ECSEnabled {
return ecszap.WrapCore(core)
}
return core
return ecszap.WrapCore(core)
}

func globalLogger() *zap.Logger {
Expand Down
3 changes: 1 addition & 2 deletions libbeat/logp/core_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func TestLogger(t *testing.T) {

TestingSetup()
exerciseLogger()
TestingSetup(AsJSON())
TestingSetup()
exerciseLogger()
}

Expand Down Expand Up @@ -152,7 +152,6 @@ func TestLoggingECSFields(t *testing.T) {
Beat: "beat1",
Level: DebugLevel,
development: true,
ECSEnabled: true,
Files: FileConfig{
Name: "beat1.log",
},
Expand Down
13 changes: 4 additions & 9 deletions libbeat/logp/encoding.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,20 +43,15 @@ type encoderCreator func(cfg zapcore.EncoderConfig) zapcore.Encoder
func buildEncoder(cfg Config) zapcore.Encoder {
var encCfg zapcore.EncoderConfig
var encCreator encoderCreator
if cfg.JSON {
encCfg = JSONEncoderConfig()
encCreator = zapcore.NewJSONEncoder
} else if cfg.ToSyslog {
if cfg.ToSyslog {
encCfg = SyslogEncoderConfig()
encCreator = zapcore.NewConsoleEncoder
} else {
encCfg = ConsoleEncoderConfig()
encCreator = zapcore.NewConsoleEncoder
encCfg = JSONEncoderConfig()
encCreator = zapcore.NewJSONEncoder
}

if cfg.ECSEnabled {
encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
}
encCfg = ecszap.ECSCompatibleEncoderConfig(encCfg)
return encCreator(encCfg)
}

Expand Down
7 changes: 0 additions & 7 deletions libbeat/logp/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,10 +52,3 @@ func ToDiscardOutput() Option {
cfg.ToStderr = false
}
}

// AsJSON specifies to log the output as JSON.
func AsJSON() Option {
return func(cfg *Config) {
cfg.JSON = true
}
}
26 changes: 4 additions & 22 deletions libbeat/tests/system/test_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,43 +25,25 @@ def assert_contains_ecs_log(self, logfile=None):
assert self.log_contains(ecs_message_log, logfile=logfile)
assert self.log_contains(ecs_log_level_log, logfile=logfile)

def assert_not_contains_ecs_log(self, logfile=None):
assert not self.log_contains(ecs_version_log, logfile=logfile)
assert not self.log_contains(ecs_timestamp_log, logfile=logfile)
assert not self.log_contains(ecs_log_level_log, logfile=logfile)

def test_console_default(self):
"""
logs to console with default format
"""
self.run_beat_with_args("mockbeat start running", logging_args=["-e"])
self.assert_not_contains_ecs_log()

def test_console_ecs(self):
"""
logs to console with ECS format
"""
self.run_beat_with_args("mockbeat start running",
logging_args=["-e"],
extra_args=["-E", "logging.json=true",
"-E", "logging.ecs=true"])
logging_args=["-e"])
self.assert_contains_ecs_log()

def test_file_default(self):
"""
logs to file with default format
"""
self.run_beat_with_args("Mockbeat is alive!",
logging_args=[],
extra_args=["-E", "logging.json=true",
"-E", "logging.ecs=false"])
self.assert_not_contains_ecs_log(logfile="logs/mockbeat")
logging_args=[])
self.assert_contains_ecs_log(logfile="logs/mockbeat")

def test_file_ecs(self):
"""
logs to file with ECS format
"""
self.run_beat_with_args("Mockbeat is alive!",
extra_args=["-E", "logging.json=true",
"-E", "logging.ecs=true"])
self.run_beat_with_args("Mockbeat is alive!")
self.assert_contains_ecs_log(logfile="logs/mockbeat")
4 changes: 2 additions & 2 deletions libbeat/tests/system/test_template.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def test_json_template(self):
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains("Loading json template from file"))
self.wait_until(lambda: self.log_contains('Template with name "bla" loaded.'))
self.wait_until(lambda: self.log_contains('Template with name \\\"bla\\\" loaded.'))
proc.check_kill_and_wait()

result = es.transport.perform_request('GET', '/_template/' + template_name)
Expand Down Expand Up @@ -149,7 +149,7 @@ def test_template_default(self):
self.render_config()
proc = self.start_beat()
self.wait_until(lambda: self.log_contains("mockbeat start running."))
self.wait_until(lambda: self.log_contains('Template with name "mockbeat-9.9.9" loaded.'))
self.wait_until(lambda: self.log_contains('Template with name \\\"mockbeat-9.9.9\\\" loaded.'))
self.wait_until(lambda: self.log_contains("PublishEvents: 1 events have been published"))
proc.check_kill_and_wait()

Expand Down
8 changes: 0 additions & 8 deletions metricbeat/metricbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2263,14 +2263,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Metricbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
2 changes: 1 addition & 1 deletion metricbeat/module/elasticsearch/test_elasticsearch.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,7 +135,7 @@ def test_xpack_cluster_stats(self):
}
}])
proc = self.start_beat()
self.wait_log_contains('"dataset": "elasticsearch.cluster.stats"')
self.wait_log_contains('\\"dataset\\": \\"elasticsearch.cluster.stats\\"')

proc.check_kill_and_wait()
self.assert_no_logged_warnings()
Expand Down
8 changes: 0 additions & 8 deletions packetbeat/packetbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1943,14 +1943,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Packetbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
8 changes: 0 additions & 8 deletions winlogbeat/winlogbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1371,14 +1371,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Winlogbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
8 changes: 0 additions & 8 deletions x-pack/auditbeat/auditbeat.reference.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1504,14 +1504,6 @@ logging.files:
# the end of the file. On rotation a new file is created, older files are untouched.
#suffix: count

# Set to true to log messages in JSON format.
#logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#logging.ecs: false

# ============================= X-Pack Monitoring ==============================
# Auditbeat can export internal metrics to a central Elasticsearch monitoring
# cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
Expand Down
5 changes: 0 additions & 5 deletions x-pack/elastic-agent/_meta/config/common.p2.yml.tmpl
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,4 @@ agent.logging.to_stderr: true
# Set to true to log messages in JSON format.
#agent.logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#agent.logging.ecs: false

{{template "providers.yml.tmpl" .}}
Original file line number Diff line number Diff line change
Expand Up @@ -197,9 +197,4 @@ agent.logging.to_stderr: true
# Set to true to log messages in JSON format.
#agent.logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#agent.logging.ecs: false

{{template "providers.yml.tmpl" .}}
5 changes: 0 additions & 5 deletions x-pack/elastic-agent/_meta/elastic-agent.yml
Original file line number Diff line number Diff line change
Expand Up @@ -191,8 +191,3 @@ agent.logging.to_stderr: true

# Set to true to log messages in JSON format.
#agent.logging.json: false

# Set to true, to log messages with minimal required Elastic Common Schema (ECS)
# information. Recommended to use in combination with `logging.json=true`
# Defaults to false.
#agent.logging.ecs: false
Loading

0 comments on commit b110f73

Please sign in to comment.