From a25115646229d68ceaf214d3e76d3684c158985d Mon Sep 17 00:00:00 2001
From: Andrew Kroh <andrew.kroh@elastic.co>
Date: Fri, 26 Jun 2020 11:13:59 -0400
Subject: [PATCH] Add additional time formats to decode_cef (#19346) (#19396)

The date formats in the CEF guide describe the time formats in terms of Java's SimpleTimeFormat class.
The `zzz` specifier covers a few additional formats than what are covered by `MST` in Go's time format.
Namely on the Go side it was missing support for offsets (e.g. +04, +0400, +04:00). This change additional
adds support for the ISO8601 `Z` time zone value (this does not strictly match the CEF guide's format).

These are the Java SimpleDateFormats in the CEF guide:

MMM dd HH:mm:ss.SSS zzz
MMM dd HH:mm:sss.SSS
MMM dd HH:mm:ss zzz
MMM dd HH:mm:ss
MMM dd yyyy HH:mm:ss.SSS zzz
MMM dd yyyy HH:mm:ss.SSS
MMM dd yyyy HH:mm:ss zzz
MMM dd yyyy HH:mm:ss

(cherry picked from commit b82829b174cc405e48f830bce33be5307406cfea)
---
 CHANGELOG.next.asciidoc                       |  1 +
 .../processors/decode_cef/cef/types.go        | 15 +++++
 .../processors/decode_cef/cef/types_test.go   | 63 +++++++++++++++++++
 3 files changed, 79 insertions(+)
 create mode 100644 x-pack/filebeat/processors/decode_cef/cef/types_test.go

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
index fa2a1e9b2ef..0fcc99264e7 100644
--- a/CHANGELOG.next.asciidoc
+++ b/CHANGELOG.next.asciidoc
@@ -490,6 +490,7 @@ field. You can revert this change by configuring tags for the module and omittin
 - Add support for v1 consumer API in Cloud Foundry input, use it by default. {pull}19125[19125]
 - Add new mode to multiline reader to aggregate constant number of lines {pull}18352[18352]
 - Explicitly set ECS version in all Filebeat modules. {pull}19198[19198]
+- Add support for timezone offsets and `Z` to decode_cef timestamp parser. {pull}19346[19346]
 
 *Heartbeat*
 
diff --git a/x-pack/filebeat/processors/decode_cef/cef/types.go b/x-pack/filebeat/processors/decode_cef/cef/types.go
index 6ef0b830622..c2c6776dcdb 100644
--- a/x-pack/filebeat/processors/decode_cef/cef/types.go
+++ b/x-pack/filebeat/processors/decode_cef/cef/types.go
@@ -103,18 +103,33 @@ func toMACAddress(v string) (string, error) {
 var timeLayouts = []string{
 	// MMM dd HH:mm:ss.SSS zzz
 	"Jan _2 15:04:05.000 MST",
+	"Jan _2 15:04:05.000 Z0700",
+	"Jan _2 15:04:05.000 Z07:00",
+
 	// MMM dd HH:mm:sss.SSS
 	"Jan _2 15:04:05.000",
+
 	// MMM dd HH:mm:ss zzz
 	"Jan _2 15:04:05 MST",
+	"Jan _2 15:04:05 Z0700",
+	"Jan _2 15:04:05 Z07:00",
+
 	// MMM dd HH:mm:ss
 	"Jan _2 15:04:05",
+
 	// MMM dd yyyy HH:mm:ss.SSS zzz
 	"Jan _2 2006 15:04:05.000 MST",
+	"Jan _2 2006 15:04:05.000 Z0700",
+	"Jan _2 2006 15:04:05.000 Z07:00",
+
 	// MMM dd yyyy HH:mm:ss.SSS
 	"Jan _2 2006 15:04:05.000",
+
 	// MMM dd yyyy HH:mm:ss zzz
 	"Jan _2 2006 15:04:05 MST",
+	"Jan _2 2006 15:04:05 Z0700",
+	"Jan _2 2006 15:04:05 Z07:00",
+
 	// MMM dd yyyy HH:mm:ss
 	"Jan _2 2006 15:04:05",
 }
diff --git a/x-pack/filebeat/processors/decode_cef/cef/types_test.go b/x-pack/filebeat/processors/decode_cef/cef/types_test.go
new file mode 100644
index 00000000000..142538eece4
--- /dev/null
+++ b/x-pack/filebeat/processors/decode_cef/cef/types_test.go
@@ -0,0 +1,63 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package cef
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestToTimestamp(t *testing.T) {
+	var times = []string{
+		// Unix epoch in milliseconds.
+		"1322004689000",
+
+		// MMM dd HH:mm:ss.SSS zzz
+		"Jun 23 17:37:24.000 Z",
+		"Jun 23 17:37:24.000 EST",
+		"Jun 23 17:37:24.000 +05",
+		"Jun 23 17:37:24.000 +0500",
+		"Jun 23 17:37:24.000 +05:00",
+
+		// MMM dd HH:mm:sss.SSS
+		"Jun 23 17:37:24.000",
+
+		// MMM dd HH:mm:ss zzz
+		"Jun 23 17:37:24 Z",
+		"Jun 23 17:37:24 EST",
+		"Jun 23 17:37:24 +05",
+		"Jun 23 17:37:24 +0500",
+		"Jun 23 17:37:24 +05:00",
+
+		// MMM dd HH:mm:ss
+		"Jun 23 17:37:24",
+
+		// MMM dd yyyy HH:mm:ss.SSS zzz
+		"Jun 23 2020 17:37:24.000 Z",
+		"Jun 23 2020 17:37:24.000 EST",
+		"Jun 23 2020 17:37:24.000 +05",
+		"Jun 23 2020 17:37:24.000 +0500",
+		"Jun 23 2020 17:37:24.000 +05:00",
+
+		// MMM dd yyyy HH:mm:ss.SSS
+		"Jun 23 2020 17:37:24.000",
+
+		// MMM dd yyyy HH:mm:ss zzz
+		"Jun 23 2020 17:37:24 Z",
+		"Jun 23 2020 17:37:24 EST",
+		"Jun 23 2020 17:37:24 +05",
+		"Jun 23 2020 17:37:24 +0500",
+		"Jun 23 2020 17:37:24 +05:00",
+
+		// MMM dd yyyy HH:mm:ss
+		"Jun 23 2020 17:37:24",
+	}
+
+	for _, timeValue := range times {
+		_, err := toTimestamp(timeValue)
+		assert.NoError(t, err, timeValue)
+	}
+}