Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add raw log message to log handler #7207

Closed
wants to merge 8 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rebase and rename to original
ruflin committed Aug 29, 2018
commit ba24d6a40788d74ad368f35cff909c478c3c746d
4 changes: 2 additions & 2 deletions filebeat/_meta/common.reference.p2.yml
Original file line number Diff line number Diff line change
@@ -81,9 +81,9 @@ filebeat.inputs:
#max_bytes: 10485760


# By default Filebeat adds log.message to each event as this contains the raw event.
# By default Filebeat adds log.original to each event as this contains the original log.
# This can be useful to reprocess the log files.
#raw_message: true
#original_message: true

### Recursive glob configuration

5 changes: 3 additions & 2 deletions filebeat/_meta/fields.common.yml
Original file line number Diff line number Diff line change
@@ -108,10 +108,11 @@
description: >
Logging level.

- name: log.message
- name: log.original
type: keyword
description: >
The unprocesses raw log message. This can be used for reprocessing logs.
The unprocessed original log message. This can be used for reprocessing logs.
index: false

- name: event.created
type: date
6 changes: 4 additions & 2 deletions filebeat/docs/fields.asciidoc
Original file line number Diff line number Diff line change
@@ -2684,14 +2684,16 @@ Logging level.

--

*`log.message`*::
*`log.original`*::
+
--
type: keyword

The unprocesses raw log message. This can be used for reprocessing logs.
The unprocessed original log message. This can be used for reprocessing logs.


Field is not indexed.

--

*`event.created`*::
4 changes: 2 additions & 2 deletions filebeat/filebeat.reference.yml
Original file line number Diff line number Diff line change
@@ -436,9 +436,9 @@ filebeat.inputs:
#max_bytes: 10485760


# By default Filebeat adds log.message to each event as this contains the raw event.
# By default Filebeat adds log.original to each event as this contains the original log.
# This can be useful to reprocess the log files.
#raw_message: true
#original_message: true

### Recursive glob configuration

38 changes: 0 additions & 38 deletions filebeat/harvester/reader/encode.go

This file was deleted.

26 changes: 0 additions & 26 deletions filebeat/harvester/reader/limit.go

This file was deleted.

50 changes: 0 additions & 50 deletions filebeat/harvester/reader/strip_newline.go

This file was deleted.

14 changes: 1 addition & 13 deletions filebeat/include/fields.go

Large diffs are not rendered by default.

34 changes: 17 additions & 17 deletions filebeat/input/log/config.go
Original file line number Diff line number Diff line change
@@ -43,17 +43,17 @@ var (
CleanInactive: 0,

// Input
Enabled: true,
IgnoreOlder: 0,
ScanFrequency: 10 * time.Second,
CleanRemoved: true,
HarvesterLimit: 0,
Symlinks: false,
TailFiles: false,
ScanSort: "",
ScanOrder: "asc",
RecursiveGlob: true,
RawMessage: true,
Enabled: true,
IgnoreOlder: 0,
ScanFrequency: 10 * time.Second,
CleanRemoved: true,
HarvesterLimit: 0,
Symlinks: false,
TailFiles: false,
ScanSort: "",
ScanOrder: "asc",
RecursiveGlob: true,
OriginalMessage: true,

// Harvester
BufferSize: 16 * humanize.KiByte,
@@ -97,12 +97,12 @@ type config struct {
ScanOrder string `config:"scan.order"`
ScanSort string `config:"scan.sort"`

ExcludeLines []match.Matcher `config:"exclude_lines"`
IncludeLines []match.Matcher `config:"include_lines"`
MaxBytes int `config:"max_bytes" validate:"min=0,nonzero"`
Multiline *multiline.Config `config:"multiline"`
JSON *readjson.Config `config:"json"`
RawMessage bool `config:"raw_message"`
ExcludeLines []match.Matcher `config:"exclude_lines"`
IncludeLines []match.Matcher `config:"include_lines"`
MaxBytes int `config:"max_bytes" validate:"min=0,nonzero"`
Multiline *multiline.Config `config:"multiline"`
JSON *readjson.Config `config:"json"`
OriginalMessage bool `config:"original_message"`

// Hidden on purpose, used by the docker input:
DockerJSON *struct {
4 changes: 2 additions & 2 deletions filebeat/input/log/harvester.go
Original file line number Diff line number Diff line change
@@ -306,8 +306,8 @@ func (h *Harvester) Run() error {
}
fields.DeepUpdate(message.Fields)

if h.config.RawMessage {
fields.Put("log.message", string(message.Raw))
if h.config.OriginalMessage {
fields.Put("log.message", string(message.Original))
}

// Check if json fields exist
10 changes: 5 additions & 5 deletions filebeat/reader/message.go
Original file line number Diff line number Diff line change
@@ -26,11 +26,11 @@ import (
// Message represents a reader event with timestamp, content and actual number
// of bytes read from input before decoding.
type Message struct {
Ts time.Time // timestamp the content was read
Content []byte // actual content read
Raw []byte // Raw content read
Bytes int // total number of bytes read to generate the message
Fields common.MapStr // optional fields that can be added by reader
Ts time.Time // timestamp the content was read
Content []byte // actual content read
Original []byte // original content read
Bytes int // total number of bytes read to generate the message
Fields common.MapStr // optional fields that can be added by reader
}

// IsEmpty returns true in case the message is empty
2 changes: 1 addition & 1 deletion filebeat/reader/multiline/multiline.go
Original file line number Diff line number Diff line change
@@ -269,7 +269,7 @@ func (mlr *Reader) clear() {
func (mlr *Reader) finalize() reader.Message {
// Copy message from existing content
msg := mlr.message
msg.Raw = mlr.message.Content
msg.Original = mlr.message.Content
mlr.clear()
return msg
}
4 changes: 4 additions & 0 deletions filebeat/reader/readfile/strip_newline.go
Original file line number Diff line number Diff line change
@@ -42,6 +42,10 @@ func (p *StripNewline) Next() (reader.Message, error) {
L := message.Content
message.Content = L[:len(L)-lineEndingChars(L)]

// Also strip new lines from raw message
M := message.Original
message.Original = M[:len(M)-lineEndingChars(M)]

return message, err
}

2 changes: 1 addition & 1 deletion filebeat/tests/system/config/filebeat.yml.j2
Original file line number Diff line number Diff line change
@@ -31,7 +31,7 @@ filebeat.{{input_config | default("inputs")}}:
harvester_limit: {{harvester_limit | default(0) }}
symlinks: {{symlinks}}
pipeline: {{pipeline}}
raw_message: {{raw_message| default("true")}}
original_message: {{original_message| default("true")}}
{%- if input_processors %}
processors:
{%- for processor in input_processors %}
12 changes: 6 additions & 6 deletions filebeat/tests/system/test_harvester.py
Original file line number Diff line number Diff line change
@@ -819,9 +819,9 @@ def test_decode_error(self):
output = self.read_output_json()
assert output[2]["message"] == "hello world2"

def test_raw_message_enabled(self):
def test_original_message_enabled(self):
"""
Test raw message enabled for json use case
Test original message enabled for json use case
"""
self.render_config_template(
path=os.path.abspath(self.working_dir) + "/log/*",
@@ -846,16 +846,16 @@ def test_raw_message_enabled(self):

assert output[0]["log.message"] == message

def test_raw_message_disabled(self):
def test_original_message_disabled(self):
"""
Test raw message enabled for json use case
Test original message enabled for json use case
"""
self.render_config_template(
path=os.path.abspath(self.working_dir) + "/log/*",
json=dict(
keys_under_root=True,
),
raw_message=False,
original_message=False,
)
os.mkdir(self.working_dir + "/log/")
logfile = self.working_dir + "/log/test.log"
@@ -872,4 +872,4 @@ def test_raw_message_disabled(self):
output = self.read_output()
assert len(output) == 1

assert "log.message" not in output[0]
assert "log.original" not in output[0]
4 changes: 2 additions & 2 deletions filebeat/tests/system/test_json.py
Original file line number Diff line number Diff line change
@@ -43,7 +43,7 @@ def test_docker_logs_filtering(self):
path=os.path.abspath(self.working_dir) + "/log/*",
json=dict(message_key="log", keys_under_root=True),
exclude_lines=["windows"],
raw_message=False,
original_message=False,
)

os.mkdir(self.working_dir + "/log/")
@@ -76,7 +76,7 @@ def test_docker_logs_multiline(self):
pattern="^\[log\]",
match="after",
negate="true",
raw_message=False,
original_message=False,
)

os.mkdir(self.working_dir + "/log/")