diff --git a/SystemdJournal2Gelf.go b/SystemdJournal2Gelf.go index 447a2e9..5b0cb9b 100644 --- a/SystemdJournal2Gelf.go +++ b/SystemdJournal2Gelf.go @@ -43,44 +43,15 @@ type SystemdJournalEntry struct { FullMessage string `json:"-"` } -// Strip date from message-content. Use named subpatterns to override other fields -var messageReplace = map[string]*regexp.Regexp{ - "*": regexp.MustCompile("^20[0-9][0-9][/\\-][01][0-9][/\\-][0123][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9][,0-9]{0-3} "), - "nginx": regexp.MustCompile("\\[(?P[a-z]+)\\] "), - "java": regexp.MustCompile("(?P[A-Z]+): "), - "mysqld": regexp.MustCompile("^[0-9]+ \\[(?P[A-Z][a-z]+)\\] "), - "searchd": regexp.MustCompile("^\\[([A-Z][a-z]{2} ){2} [0-9]+ [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\\.[0-9]{3} 20[0-9][0-9]\\] \\[[ 0-9]+\\] "), - "jenkins": regexp.MustCompile("^[A-Z][a-z]{2} [01][0-9], 20[0-9][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [AP]M "), - "php-fpm": regexp.MustCompile("^pool [a-z_0-9\\[\\]\\-]+: "), - "syncthing": regexp.MustCompile("^\\[[0-9A-Z]{5}\\] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] (?PINFO): "), -} - -var priorities = map[string]int32{ - "emergency": 0, - "emerg": 0, - "alert": 1, - "critical": 2, - "crit": 2, - "error": 3, - "err": 3, - "warning": 4, - "warn": 4, - "notice": 5, - "info": 6, - "debug": 7, -} +// Strip date from message-content +var startsWithTimestamp = regexp.MustCompile("^20[0-9][0-9][/\\-][01][0-9][/\\-][0123][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9][,0-9]{0,3} ") func (this *SystemdJournalEntry) toGelf() *gelf.Message { var extra = map[string]interface{}{ - "Boot_id": this.Boot_id, - "Pid": this.Pid, - "Uid": this.Uid, - } - - // php-fpm refuses to fill identifier - facility := this.Syslog_identifier - if "php-fpm" == this.Comm { - facility = this.Comm + "Boot_id": this.Boot_id, + "Pid": this.Pid, + "Uid": this.Uid, + "Systemd_unit": this.Systemd_unit, } if this.isJsonMessage() { @@ -107,40 +78,11 @@ func (this *SystemdJournalEntry) toGelf() *gelf.Message { Full: this.FullMessage, TimeUnix: float64(this.Realtime_timestamp) / 1000 / 1000, Level: this.Priority, - Facility: facility, + Facility: this.Syslog_identifier, Extra: extra, } } -// FIXME remove in favor of Graylogs extractors? -func (this *SystemdJournalEntry) process() { - // Replace generic timestamp - this.Message = messageReplace["*"].ReplaceAllString(this.Message, "") - - re := messageReplace[this.Syslog_identifier] - if nil == re { - re = messageReplace[this.Comm] - } - - if nil == re { - return - } - - m := re.FindStringSubmatch(this.Message) - if m == nil { - return - } - - // Store subpatterns in fields - for idx, key := range re.SubexpNames() { - if "Priority" == key { - this.Priority = priorities[strings.ToLower(m[idx])] - } - } - - this.Message = re.ReplaceAllString(this.Message, "") -} - // Custom wrapper to support unprintable chars in message func (this *SystemdJournalEntry) UnmarshalJSON(data []byte) error { // use an alias to prevent recursion @@ -148,6 +90,8 @@ func (this *SystemdJournalEntry) UnmarshalJSON(data []byte) error { aux := (*entryAlias)(this) if err := json.Unmarshal(data, &aux); err == nil { + this.Message = startsWithTimestamp.ReplaceAllString(this.Message, "") + return nil } else if ute, ok := err.(*json.UnmarshalTypeError); ok && ute.Field == "MESSAGE" && ute.Value == "array" { // Include brackets, which is why we subtract and add by one @@ -263,8 +207,6 @@ func main() { panic("could not parse journal output: " + err.Error()) } - entry.process() - pending.Push(entry) // Prevent saturation and throttling diff --git a/SystemdJournal2Gelf_test.go b/SystemdJournal2Gelf_test.go index 7f4c1d4..c302414 100644 --- a/SystemdJournal2Gelf_test.go +++ b/SystemdJournal2Gelf_test.go @@ -31,7 +31,7 @@ func TestUnmarshalEntry(t *testing.T) { AssertEquals(t, int32(5), gelf.Level) AssertEquals(t, "kernel", gelf.Facility) - AssertEquals(t, 3, len(gelf.Extra)) + AssertEquals(t, 4, len(gelf.Extra)) AssertEquals(t, "61c0e40c739f4f009c785cef13b46e17", gelf.Extra["Boot_id"]) AssertEquals(t, "99", gelf.Extra["Uid"]) AssertEquals(t, "1234", gelf.Extra["Pid"]) @@ -55,7 +55,7 @@ func TestJsonMessageOverridesNormalProperties(t *testing.T) { AssertEquals(t, "actually something else", gelf.Short) AssertEquals(t, "additional data", gelf.Full) AssertEquals(t, "kernel", gelf.Facility) - AssertEquals(t, 3, len(gelf.Extra)) + AssertEquals(t, 4, len(gelf.Extra)) } func TestJsonMessageIncludeDataInExtra(t *testing.T) { @@ -74,7 +74,7 @@ func TestJsonMessageIncludeDataInExtra(t *testing.T) { AssertEquals(t, "machine.nl", gelf.Host) AssertEquals(t, "actually something else", gelf.Short) AssertEquals(t, "kernel", gelf.Facility) - AssertEquals(t, 4, len(gelf.Extra)) + AssertEquals(t, 5, len(gelf.Extra)) AssertEquals(t, "things and stuff and more like that", gelf.Extra["stuff"]) } @@ -97,6 +97,28 @@ func TestUnmarshalUnprintableEntry(t *testing.T) { } +func TestDateStrippedFromMessage(t *testing.T) { + entry := SystemdJournalEntry{} + + err := json.Unmarshal([]byte(`{ + "_HOSTNAME" : "machine.nl", + "MESSAGE" : "2019-04-04 13:20:27 15024 [Warning] Aborted connection", + "SYSLOG_IDENTIFIER" : "mysqld" + }`), &entry) + + AssertNotError(t, err) + + gelf := entry.toGelf() + + AssertEquals(t, "machine.nl", gelf.Host) + AssertEquals(t, "15024 [Warning] Aborted connection", gelf.Short) + AssertEquals(t, "", gelf.Full) + AssertEquals(t, "mysqld", gelf.Facility) + AssertEquals(t, 4, len(gelf.Extra)) +} + +// asserts + func AssertEquals(t *testing.T, expected, actual interface{}) { if expected != actual { t.Errorf("AssertEquals: %[1]T(%#[1]v) does not match %[2]T(%#[2]v))", actual, expected)