Remove messageReplaces with a single date-strip regexp, add Systemd_unit

let graylog extractors do any other processing
parse-nl · Apr 4, 2019 · ced89e0 · ced89e0
1 parent 32cd91d
commit ced89e0
Show file tree

Hide file tree

Showing 2 changed files with 34 additions and 70 deletions.
diff --git a/SystemdJournal2Gelf.go b/SystemdJournal2Gelf.go
@@ -43,44 +43,15 @@ type SystemdJournalEntry struct {
 	FullMessage               string `json:"-"`
 }
 
-// Strip date from message-content. Use named subpatterns to override other fields
-var messageReplace = map[string]*regexp.Regexp{
-	"*":         regexp.MustCompile("^20[0-9][0-9][/\\-][01][0-9][/\\-][0123][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9][,0-9]{0-3} "),
-	"nginx":     regexp.MustCompile("\\[(?P<Priority>[a-z]+)\\] "),
-	"java":      regexp.MustCompile("(?P<Priority>[A-Z]+): "),
-	"mysqld":    regexp.MustCompile("^[0-9]+ \\[(?P<Priority>[A-Z][a-z]+)\\] "),
-	"searchd":   regexp.MustCompile("^\\[([A-Z][a-z]{2} ){2} [0-9]+ [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\\.[0-9]{3} 20[0-9][0-9]\\] \\[[ 0-9]+\\] "),
-	"jenkins":   regexp.MustCompile("^[A-Z][a-z]{2} [01][0-9], 20[0-9][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9] [AP]M "),
-	"php-fpm":   regexp.MustCompile("^pool [a-z_0-9\\[\\]\\-]+: "),
-	"syncthing": regexp.MustCompile("^\\[[0-9A-Z]{5}\\] [0-2][0-9]:[0-5][0-9]:[0-5][0-9] (?P<Priority>INFO): "),
-}
-
-var priorities = map[string]int32{
-	"emergency": 0,
-	"emerg":     0,
-	"alert":     1,
-	"critical":  2,
-	"crit":      2,
-	"error":     3,
-	"err":       3,
-	"warning":   4,
-	"warn":      4,
-	"notice":    5,
-	"info":      6,
-	"debug":     7,
-}
+// Strip date from message-content
+var startsWithTimestamp = regexp.MustCompile("^20[0-9][0-9][/\\-][01][0-9][/\\-][0123][0-9] [0-2]?[0-9]:[0-5][0-9]:[0-5][0-9][,0-9]{0,3} ")
 
 func (this *SystemdJournalEntry) toGelf() *gelf.Message {
 	var extra = map[string]interface{}{
-		"Boot_id": this.Boot_id,
-		"Pid":     this.Pid,
-		"Uid":     this.Uid,
-	}
-
-	// php-fpm refuses to fill identifier
-	facility := this.Syslog_identifier
-	if "php-fpm" == this.Comm {
-		facility = this.Comm
+		"Boot_id":      this.Boot_id,
+		"Pid":          this.Pid,
+		"Uid":          this.Uid,
+		"Systemd_unit": this.Systemd_unit,
 	}
 
 	if this.isJsonMessage() {
@@ -107,47 +78,20 @@ func (this *SystemdJournalEntry) toGelf() *gelf.Message {
 		Full:     this.FullMessage,
 		TimeUnix: float64(this.Realtime_timestamp) / 1000 / 1000,
 		Level:    this.Priority,
-		Facility: facility,
+		Facility: this.Syslog_identifier,
 		Extra:    extra,
 	}
 }
 
-// FIXME remove in favor of Graylogs extractors?
-func (this *SystemdJournalEntry) process() {
-	// Replace generic timestamp
-	this.Message = messageReplace["*"].ReplaceAllString(this.Message, "")
-
-	re := messageReplace[this.Syslog_identifier]
-	if nil == re {
-		re = messageReplace[this.Comm]
-	}
-
-	if nil == re {
-		return
-	}
-
-	m := re.FindStringSubmatch(this.Message)
-	if m == nil {
-		return
-	}
-
-	// Store subpatterns in fields
-	for idx, key := range re.SubexpNames() {
-		if "Priority" == key {
-			this.Priority = priorities[strings.ToLower(m[idx])]
-		}
-	}
-
-	this.Message = re.ReplaceAllString(this.Message, "")
-}
-
 // Custom wrapper to support unprintable chars in message
 func (this *SystemdJournalEntry) UnmarshalJSON(data []byte) error {
 	// use an alias to prevent recursion
 	type entryAlias SystemdJournalEntry
 	aux := (*entryAlias)(this)
 
 	if err := json.Unmarshal(data, &aux); err == nil {
+		this.Message = startsWithTimestamp.ReplaceAllString(this.Message, "")
+
 		return nil
 	} else if ute, ok := err.(*json.UnmarshalTypeError); ok && ute.Field == "MESSAGE" && ute.Value == "array" {
 		// Include brackets, which is why we subtract and add by one
@@ -263,8 +207,6 @@ func main() {
 			panic("could not parse journal output: " + err.Error())
 		}
 
-		entry.process()
-
 		pending.Push(entry)
 
 		// Prevent saturation and throttling

diff --git a/SystemdJournal2Gelf_test.go b/SystemdJournal2Gelf_test.go
@@ -31,7 +31,7 @@ func TestUnmarshalEntry(t *testing.T) {
 	AssertEquals(t, int32(5), gelf.Level)
 	AssertEquals(t, "kernel", gelf.Facility)
 
-	AssertEquals(t, 3, len(gelf.Extra))
+	AssertEquals(t, 4, len(gelf.Extra))
 	AssertEquals(t, "61c0e40c739f4f009c785cef13b46e17", gelf.Extra["Boot_id"])
 	AssertEquals(t, "99", gelf.Extra["Uid"])
 	AssertEquals(t, "1234", gelf.Extra["Pid"])
@@ -55,7 +55,7 @@ func TestJsonMessageOverridesNormalProperties(t *testing.T) {
 	AssertEquals(t, "actually something else", gelf.Short)
 	AssertEquals(t, "additional data", gelf.Full)
 	AssertEquals(t, "kernel", gelf.Facility)
-	AssertEquals(t, 3, len(gelf.Extra))
+	AssertEquals(t, 4, len(gelf.Extra))
 }
 
 func TestJsonMessageIncludeDataInExtra(t *testing.T) {
@@ -74,7 +74,7 @@ func TestJsonMessageIncludeDataInExtra(t *testing.T) {
 	AssertEquals(t, "machine.nl", gelf.Host)
 	AssertEquals(t, "actually something else", gelf.Short)
 	AssertEquals(t, "kernel", gelf.Facility)
-	AssertEquals(t, 4, len(gelf.Extra))
+	AssertEquals(t, 5, len(gelf.Extra))
 	AssertEquals(t, "things and stuff and more like that", gelf.Extra["stuff"])
 }
 
@@ -97,6 +97,28 @@ func TestUnmarshalUnprintableEntry(t *testing.T) {
 
 }
 
+func TestDateStrippedFromMessage(t *testing.T) {
+	entry := SystemdJournalEntry{}
+
+	err := json.Unmarshal([]byte(`{
+        "_HOSTNAME" : "machine.nl",
+        "MESSAGE" : "2019-04-04 13:20:27 15024 [Warning] Aborted connection",
+        "SYSLOG_IDENTIFIER" : "mysqld"
+	}`), &entry)
+
+	AssertNotError(t, err)
+
+	gelf := entry.toGelf()
+
+	AssertEquals(t, "machine.nl", gelf.Host)
+	AssertEquals(t, "15024 [Warning] Aborted connection", gelf.Short)
+	AssertEquals(t, "", gelf.Full)
+	AssertEquals(t, "mysqld", gelf.Facility)
+	AssertEquals(t, 4, len(gelf.Extra))
+}
+
+// asserts
+
 func AssertEquals(t *testing.T, expected, actual interface{}) {
 	if expected != actual {
 		t.Errorf("AssertEquals: %[1]T(%#[1]v) does not match %[2]T(%#[2]v))", actual, expected)